REDACTED
", + "date_joined": "2011-06-25T20:53:48+00:00", + "id": 18, + "name": "Sethisto", + "num_blog_posts": 0, + "num_followers": 143, + "num_stories": 1, + "url": "https://www.fimfiction.net/user/18/Sethisto" + }, + "chapters": [ + { + "chapter_number": 1, + "date_modified": "2014-01-28T11:25:52+00:00", + "date_published": "2011-07-08T18:04:11+00:00", + "id": 10, + "num_views": 10498, + "num_words": 321, + "published": true, + "title": "Chapter 1", + "url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1" + } + ], + "color": { + "hex": "3e3e7e", + "rgb": [ + 62, + 62, + 126 + ] + }, + "completion_status": "incomplete", + "content_rating": "everyone", + "cover_image": { + "full": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-full", + "large": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-large", + "medium": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-medium", + "thumbnail": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-tiny" + }, + "date_modified": null, + "date_published": "2011-07-08T18:04:11+00:00", + "date_updated": "2011-06-25T21:05:53+00:00", + "description_html": "
REDACTED
", + "id": 9, + "num_chapters": 1, + "num_comments": 228, + "num_dislikes": 52, + "num_likes": 398, + "num_views": 10497, + "num_words": 321, + "prequel": null, + "published": true, + "rating": 88, + "short_description": "REDACTED", + "status": "visible", + "submitted": true, + "tags": [ + { + "id": 20, + "name": "Trixie", + "old_id": "c:21", + "type": "character", + "url": "https://www.fimfiction.net/tag/trixie" + }, + { + "id": 6, + "name": "Twilight Sparkle", + "old_id": "c:7", + "type": "character", + "url": "https://www.fimfiction.net/tag/twilight-sparkle" + }, + { + "id": 234, + "name": "Random", + "old_id": "g:random", + "type": "genre", + "url": "https://www.fimfiction.net/tag/random" + }, + { + "id": 120, + "name": "Romance", + "old_id": "g:romance", + "type": "genre", + "url": "https://www.fimfiction.net/tag/romance" + }, + { + "id": 4, + "name": "My Little Pony: Friendship is Magic", + "old_id": "", + "type": "series", + "url": "https://www.fimfiction.net/tag/mlp-fim" + } + ], + "title": "The Greatest Equine Who has Ever Lived!", + "total_num_views": 10497, + "url": "https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived" + } + } + } +} diff --git a/tests/fetchers/test_fimfarchive.py b/tests/fetchers/test_fimfarchive.py index 26cfb1f..be67b27 100644 --- a/tests/fetchers/test_fimfarchive.py +++ b/tests/fetchers/test_fimfarchive.py @@ -5,7 +5,7 @@ Fimfarchive fetcher tests. # # Fimfarchive, preserves stories from Fimfiction. -# Copyright (C) 2015 Joakim Soderlund +# Copyright (C) 2019 Joakim Soderlund # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,16 +22,153 @@ Fimfarchive fetcher tests. # +import json +from io import BytesIO +from typing import Any, Dict, List +from zipfile import ZipFile + +import arrow import pytest from fimfarchive.exceptions import InvalidStoryError, StorySourceError -from fimfarchive.fetchers import FimfarchiveFetcher +from fimfarchive.fetchers import Fetcher, FimfarchiveFetcher +from fimfarchive.stories import Story +from fimfarchive.utils import JayWalker VALID_STORY_KEY = 9 INVALID_STORY_KEY = 7 -FIMFARCHIVE_PATH = 'fimfarchive.zip' + +@pytest.fixture(scope='module') +def data(): + """ + Returns test data from JSON. + """ + path = f'{__file__[:-3]}.json' + + with open(path, 'rt') as fobj: + return json.load(fobj) + + +class Redactor(JayWalker): + """ + Redacts samples. + """ + + def handle(self, data, key, value) -> None: + if str(key).endswith('_html'): + data[key] = '
REDACTED
' + elif key == 'short_description': + data[key] = "REDACTED" + else: + self.walk(value) + + +class FimfarchiveFetcherSampler: + """ + Generates a sample archive for tests. + + Samples must be manually inspected for correctness. + """ + + def __init__(self, fetcher: Fetcher, *keys: int) -> None: + """ + Constructor. + + Args: + fetcher: The fetcher to fetch from. + *keys: The stories to sample. + """ + self.redactor = Redactor() + self.stories = [self.sample(fetcher, key) for key in keys] + + def sample(self, fetcher: Fetcher, key: int) -> Story: + """ + Returns a redacted story sample. + """ + story = fetcher.fetch(key) + story = story.merge(data=b'REDACTED') + self.redactor.walk(story.meta) + + return story + + @property + def files(self) -> List[Dict[str, str]]: + """ + Returns a list of story data files. + """ + files = [] + + for story in self.stories: + files.append({ + 'name': story.meta['archive']['path'], + 'text': story.data.decode(), + }) + + return files + + @property + def about(self) -> Dict[str, str]: + """ + Returns the about file dictionary. + """ + today = arrow.utcnow() + fmt = 'YYYYMMDD' + + return { + 'version': today.shift(days=-1).format(fmt), + 'start': today.shift(days=-9).format(fmt), + 'end': today.shift(days=-2).format(fmt), + } + + @property + def index(self) -> Dict[int, Any]: + """ + Returns the index file dictionary. + """ + return { + story.key: story.meta + for story in self.stories + } + + @property + def archive(self) -> Dict[str, Any]: + """ + Returns all the sample content. + """ + return { + 'about': self.about, + 'files': self.files, + 'index': self.index, + } + + def __str__(self) -> str: + """ + Serializes all samples. + """ + return json.dumps( + {'archive': self.archive}, + ensure_ascii=False, + sort_keys=True, + indent=4, + ) + + +def serialize(obj: Dict) -> str: + """ + Serializes into JSON readable by the fetcher. + """ + entries = [] + + for key, value in obj.items(): + data = json.dumps(value, sort_keys=True) + entries.append(f'"{key}": {data}') + + joined = ',\n'.join(entries) + output = '\n'.join(('{', joined, '}', '')) + + return output class TestFimfarchiveFetcher: @@ -40,19 +177,38 @@ class TestFimfarchiveFetcher: """ @pytest.fixture(scope='module') - def fetcher(self): + def archive(self, data): + """ + Returns the archive as a byte stream. + """ + stream = BytesIO() + + zobj = ZipFile(stream, 'w') + archive = data['archive'] + + for entry in archive['files']: + zobj.writestr(entry['name'], entry['text']) + + zobj.writestr('readme.pdf', 'REDACTED') + zobj.writestr('about.json', serialize(archive['about'])) + zobj.writestr('index.json', serialize(archive['index'])) + zobj.close() + + return stream + + @pytest.fixture() + def fetcher(self, archive): """ Returns the fetcher instance to test. """ - with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher: + with FimfarchiveFetcher(archive) as fetcher: yield fetcher - def test_closed_fetcher_raises_exception(self): + def test_closed_fetcher_raises_exception(self, fetcher): """ Tests `StorySourceError` is raised when fetcher is closed. """ - with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher: - fetcher.fetch_meta(VALID_STORY_KEY) + fetcher.close() with pytest.raises(StorySourceError): fetcher.fetch_meta(VALID_STORY_KEY) @@ -86,9 +242,9 @@ class TestFimfarchiveFetcher: fetcher.fetch_data(INVALID_STORY_KEY) @pytest.mark.parametrize('attr', ('archive', 'index', 'paths')) - def test_close_when_missing_attribute(self, attr): + def test_close_when_missing_attribute(self, fetcher, attr): """ Tests close works even after partial initialization. """ - with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher: - delattr(fetcher, attr) + delattr(fetcher, attr) + fetcher.close() diff --git a/tox.ini b/tox.ini index 61f00f8..3ff9153 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,6 @@ commands = [pytest] addopts = - --ignore tests/fetchers/test_fimfarchive.py tests [flake8]