mirror of
https://github.com/JockeTF/fimfarchive.git
synced 2024-11-22 05:17:59 +01:00
Add archive sample for Fimfarchive fetcher tests
This commit is contained in:
Joakim Soderlund
parent
1accc8c750
commit
314923a4de
3 changed files with 302 additions and 12 deletions
135
tests/fetchers/test_fimfarchive.json
Normal file
135
tests/fetchers/test_fimfarchive.json
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
{
|
||||
"archive": {
|
||||
"about": {
|
||||
"end": "20190315",
|
||||
"start": "20190308",
|
||||
"version": "20190316"
|
||||
},
|
||||
"files": [
|
||||
{
|
||||
"name": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub",
|
||||
"text": "REDACTED"
|
||||
}
|
||||
],
|
||||
"index": {
|
||||
"9": {
|
||||
"archive": {
|
||||
"date_checked": "2019-01-30T17:25:59.374016+00:00",
|
||||
"date_created": null,
|
||||
"date_fetched": "2019-01-30T17:25:59.374016+00:00",
|
||||
"date_updated": "2017-11-01T21:27:29.364912+00:00",
|
||||
"path": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub"
|
||||
},
|
||||
"author": {
|
||||
"avatar": {
|
||||
"128": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-128",
|
||||
"160": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-160",
|
||||
"192": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-192",
|
||||
"256": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-256",
|
||||
"32": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-32",
|
||||
"320": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-320",
|
||||
"384": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-384",
|
||||
"48": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-48",
|
||||
"512": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-512",
|
||||
"64": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-64",
|
||||
"96": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-96"
|
||||
},
|
||||
"bio_html": "<p>REDACTED<p>",
|
||||
"date_joined": "2011-06-25T20:53:48+00:00",
|
||||
"id": 18,
|
||||
"name": "Sethisto",
|
||||
"num_blog_posts": 0,
|
||||
"num_followers": 143,
|
||||
"num_stories": 1,
|
||||
"url": "https://www.fimfiction.net/user/18/Sethisto"
|
||||
},
|
||||
"chapters": [
|
||||
{
|
||||
"chapter_number": 1,
|
||||
"date_modified": "2014-01-28T11:25:52+00:00",
|
||||
"date_published": "2011-07-08T18:04:11+00:00",
|
||||
"id": 10,
|
||||
"num_views": 10498,
|
||||
"num_words": 321,
|
||||
"published": true,
|
||||
"title": "Chapter 1",
|
||||
"url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1"
|
||||
}
|
||||
],
|
||||
"color": {
|
||||
"hex": "3e3e7e",
|
||||
"rgb": [
|
||||
62,
|
||||
62,
|
||||
126
|
||||
]
|
||||
},
|
||||
"completion_status": "incomplete",
|
||||
"content_rating": "everyone",
|
||||
"cover_image": {
|
||||
"full": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-full",
|
||||
"large": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-large",
|
||||
"medium": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-medium",
|
||||
"thumbnail": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-tiny"
|
||||
},
|
||||
"date_modified": null,
|
||||
"date_published": "2011-07-08T18:04:11+00:00",
|
||||
"date_updated": "2011-06-25T21:05:53+00:00",
|
||||
"description_html": "<p>REDACTED<p>",
|
||||
"id": 9,
|
||||
"num_chapters": 1,
|
||||
"num_comments": 228,
|
||||
"num_dislikes": 52,
|
||||
"num_likes": 398,
|
||||
"num_views": 10497,
|
||||
"num_words": 321,
|
||||
"prequel": null,
|
||||
"published": true,
|
||||
"rating": 88,
|
||||
"short_description": "REDACTED",
|
||||
"status": "visible",
|
||||
"submitted": true,
|
||||
"tags": [
|
||||
{
|
||||
"id": 20,
|
||||
"name": "Trixie",
|
||||
"old_id": "c:21",
|
||||
"type": "character",
|
||||
"url": "https://www.fimfiction.net/tag/trixie"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"name": "Twilight Sparkle",
|
||||
"old_id": "c:7",
|
||||
"type": "character",
|
||||
"url": "https://www.fimfiction.net/tag/twilight-sparkle"
|
||||
},
|
||||
{
|
||||
"id": 234,
|
||||
"name": "Random",
|
||||
"old_id": "g:random",
|
||||
"type": "genre",
|
||||
"url": "https://www.fimfiction.net/tag/random"
|
||||
},
|
||||
{
|
||||
"id": 120,
|
||||
"name": "Romance",
|
||||
"old_id": "g:romance",
|
||||
"type": "genre",
|
||||
"url": "https://www.fimfiction.net/tag/romance"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"name": "My Little Pony: Friendship is Magic",
|
||||
"old_id": "",
|
||||
"type": "series",
|
||||
"url": "https://www.fimfiction.net/tag/mlp-fim"
|
||||
}
|
||||
],
|
||||
"title": "The Greatest Equine Who has Ever Lived!",
|
||||
"total_num_views": 10497,
|
||||
"url": "https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ Fimfarchive fetcher tests.
|
|||
|
||||
#
|
||||
# Fimfarchive, preserves stories from Fimfiction.
|
||||
# Copyright (C) 2015 Joakim Soderlund
|
||||
# Copyright (C) 2019 Joakim Soderlund
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -22,16 +22,153 @@ Fimfarchive fetcher tests.
|
|||
#
|
||||
|
||||
|
||||
import json
|
||||
from io import BytesIO
|
||||
from typing import Any, Dict, List
|
||||
from zipfile import ZipFile
|
||||
|
||||
import arrow
|
||||
import pytest
|
||||
|
||||
from fimfarchive.exceptions import InvalidStoryError, StorySourceError
|
||||
from fimfarchive.fetchers import FimfarchiveFetcher
|
||||
from fimfarchive.fetchers import Fetcher, FimfarchiveFetcher
|
||||
from fimfarchive.stories import Story
|
||||
from fimfarchive.utils import JayWalker
|
||||
|
||||
|
||||
VALID_STORY_KEY = 9
|
||||
INVALID_STORY_KEY = 7
|
||||
|
||||
FIMFARCHIVE_PATH = 'fimfarchive.zip'
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
def data():
|
||||
"""
|
||||
Returns test data from JSON.
|
||||
"""
|
||||
path = f'{__file__[:-3]}.json'
|
||||
|
||||
with open(path, 'rt') as fobj:
|
||||
return json.load(fobj)
|
||||
|
||||
|
||||
class Redactor(JayWalker):
|
||||
"""
|
||||
Redacts samples.
|
||||
"""
|
||||
|
||||
def handle(self, data, key, value) -> None:
|
||||
if str(key).endswith('_html'):
|
||||
data[key] = '<p>REDACTED<p>'
|
||||
elif key == 'short_description':
|
||||
data[key] = "REDACTED"
|
||||
else:
|
||||
self.walk(value)
|
||||
|
||||
|
||||
class FimfarchiveFetcherSampler:
|
||||
"""
|
||||
Generates a sample archive for tests.
|
||||
|
||||
Samples must be manually inspected for correctness.
|
||||
"""
|
||||
|
||||
def __init__(self, fetcher: Fetcher, *keys: int) -> None:
|
||||
"""
|
||||
Constructor.
|
||||
|
||||
Args:
|
||||
fetcher: The fetcher to fetch from.
|
||||
*keys: The stories to sample.
|
||||
"""
|
||||
self.redactor = Redactor()
|
||||
self.stories = [self.sample(fetcher, key) for key in keys]
|
||||
|
||||
def sample(self, fetcher: Fetcher, key: int) -> Story:
|
||||
"""
|
||||
Returns a redacted story sample.
|
||||
"""
|
||||
story = fetcher.fetch(key)
|
||||
story = story.merge(data=b'REDACTED')
|
||||
self.redactor.walk(story.meta)
|
||||
|
||||
return story
|
||||
|
||||
@property
|
||||
def files(self) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Returns a list of story data files.
|
||||
"""
|
||||
files = []
|
||||
|
||||
for story in self.stories:
|
||||
files.append({
|
||||
'name': story.meta['archive']['path'],
|
||||
'text': story.data.decode(),
|
||||
})
|
||||
|
||||
return files
|
||||
|
||||
@property
|
||||
def about(self) -> Dict[str, str]:
|
||||
"""
|
||||
Returns the about file dictionary.
|
||||
"""
|
||||
today = arrow.utcnow()
|
||||
fmt = 'YYYYMMDD'
|
||||
|
||||
return {
|
||||
'version': today.shift(days=-1).format(fmt),
|
||||
'start': today.shift(days=-9).format(fmt),
|
||||
'end': today.shift(days=-2).format(fmt),
|
||||
}
|
||||
|
||||
@property
|
||||
def index(self) -> Dict[int, Any]:
|
||||
"""
|
||||
Returns the index file dictionary.
|
||||
"""
|
||||
return {
|
||||
story.key: story.meta
|
||||
for story in self.stories
|
||||
}
|
||||
|
||||
@property
|
||||
def archive(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Returns all the sample content.
|
||||
"""
|
||||
return {
|
||||
'about': self.about,
|
||||
'files': self.files,
|
||||
'index': self.index,
|
||||
}
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""
|
||||
Serializes all samples.
|
||||
"""
|
||||
return json.dumps(
|
||||
{'archive': self.archive},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
)
|
||||
|
||||
|
||||
def serialize(obj: Dict) -> str:
|
||||
"""
|
||||
Serializes into JSON readable by the fetcher.
|
||||
"""
|
||||
entries = []
|
||||
|
||||
for key, value in obj.items():
|
||||
data = json.dumps(value, sort_keys=True)
|
||||
entries.append(f'"{key}": {data}')
|
||||
|
||||
joined = ',\n'.join(entries)
|
||||
output = '\n'.join(('{', joined, '}', ''))
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class TestFimfarchiveFetcher:
|
||||
|
|
@ -40,19 +177,38 @@ class TestFimfarchiveFetcher:
|
|||
"""
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
def fetcher(self):
|
||||
def archive(self, data):
|
||||
"""
|
||||
Returns the archive as a byte stream.
|
||||
"""
|
||||
stream = BytesIO()
|
||||
|
||||
zobj = ZipFile(stream, 'w')
|
||||
archive = data['archive']
|
||||
|
||||
for entry in archive['files']:
|
||||
zobj.writestr(entry['name'], entry['text'])
|
||||
|
||||
zobj.writestr('readme.pdf', 'REDACTED')
|
||||
zobj.writestr('about.json', serialize(archive['about']))
|
||||
zobj.writestr('index.json', serialize(archive['index']))
|
||||
zobj.close()
|
||||
|
||||
return stream
|
||||
|
||||
@pytest.fixture()
|
||||
def fetcher(self, archive):
|
||||
"""
|
||||
Returns the fetcher instance to test.
|
||||
"""
|
||||
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
|
||||
with FimfarchiveFetcher(archive) as fetcher:
|
||||
yield fetcher
|
||||
|
||||
def test_closed_fetcher_raises_exception(self):
|
||||
def test_closed_fetcher_raises_exception(self, fetcher):
|
||||
"""
|
||||
Tests `StorySourceError` is raised when fetcher is closed.
|
||||
"""
|
||||
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
|
||||
fetcher.fetch_meta(VALID_STORY_KEY)
|
||||
fetcher.close()
|
||||
|
||||
with pytest.raises(StorySourceError):
|
||||
fetcher.fetch_meta(VALID_STORY_KEY)
|
||||
|
|
@ -86,9 +242,9 @@ class TestFimfarchiveFetcher:
|
|||
fetcher.fetch_data(INVALID_STORY_KEY)
|
||||
|
||||
@pytest.mark.parametrize('attr', ('archive', 'index', 'paths'))
|
||||
def test_close_when_missing_attribute(self, attr):
|
||||
def test_close_when_missing_attribute(self, fetcher, attr):
|
||||
"""
|
||||
Tests close works even after partial initialization.
|
||||
"""
|
||||
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
|
||||
delattr(fetcher, attr)
|
||||
fetcher.close()
|
||||
|
|
|
|||
1
tox.ini
1
tox.ini
|
|
@ -17,7 +17,6 @@ commands =
|
|||
|
||||
[pytest]
|
||||
addopts =
|
||||
--ignore tests/fetchers/test_fimfarchive.py
|
||||
tests
|
||||
|
||||
[flake8]
|
||||
|
|
|
|||
Loading…
Reference in a new issue