Add archive sample for Fimfarchive fetcher tests

tests/fetchers/test_fimfarchive.json

@@ -0,0 +1,135 @@
+{
+    "archive": {
+        "about": {
+            "end": "20190315",
+            "start": "20190308",
+            "version": "20190316"
+        },
+        "files": [
+            {
+                "name": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub",
+                "text": "REDACTED"
+            }
+        ],
+        "index": {
+            "9": {
+                "archive": {
+                    "date_checked": "2019-01-30T17:25:59.374016+00:00",
+                    "date_created": null,
+                    "date_fetched": "2019-01-30T17:25:59.374016+00:00",
+                    "date_updated": "2017-11-01T21:27:29.364912+00:00",
+                    "path": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub"
+                },
+                "author": {
+                    "avatar": {
+                        "128": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-128",
+                        "160": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-160",
+                        "192": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-192",
+                        "256": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-256",
+                        "32": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-32",
+                        "320": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-320",
+                        "384": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-384",
+                        "48": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-48",
+                        "512": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-512",
+                        "64": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-64",
+                        "96": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-96"
+                    },
+                    "bio_html": "<p>REDACTED<p>",
+                    "date_joined": "2011-06-25T20:53:48+00:00",
+                    "id": 18,
+                    "name": "Sethisto",
+                    "num_blog_posts": 0,
+                    "num_followers": 143,
+                    "num_stories": 1,
+                    "url": "https://www.fimfiction.net/user/18/Sethisto"
+                },
+                "chapters": [
+                    {
+                        "chapter_number": 1,
+                        "date_modified": "2014-01-28T11:25:52+00:00",
+                        "date_published": "2011-07-08T18:04:11+00:00",
+                        "id": 10,
+                        "num_views": 10498,
+                        "num_words": 321,
+                        "published": true,
+                        "title": "Chapter 1",
+                        "url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1"
+                    }
+                ],
+                "color": {
+                    "hex": "3e3e7e",
+                    "rgb": [
+                        62,
+                        62,
+                        126
+                    ]
+                },
+                "completion_status": "incomplete",
+                "content_rating": "everyone",
+                "cover_image": {
+                    "full": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-full",
+                    "large": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-large",
+                    "medium": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-medium",
+                    "thumbnail": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-tiny"
+                },
+                "date_modified": null,
+                "date_published": "2011-07-08T18:04:11+00:00",
+                "date_updated": "2011-06-25T21:05:53+00:00",
+                "description_html": "<p>REDACTED<p>",
+                "id": 9,
+                "num_chapters": 1,
+                "num_comments": 228,
+                "num_dislikes": 52,
+                "num_likes": 398,
+                "num_views": 10497,
+                "num_words": 321,
+                "prequel": null,
+                "published": true,
+                "rating": 88,
+                "short_description": "REDACTED",
+                "status": "visible",
+                "submitted": true,
+                "tags": [
+                    {
+                        "id": 20,
+                        "name": "Trixie",
+                        "old_id": "c:21",
+                        "type": "character",
+                        "url": "https://www.fimfiction.net/tag/trixie"
+                    },
+                    {
+                        "id": 6,
+                        "name": "Twilight Sparkle",
+                        "old_id": "c:7",
+                        "type": "character",
+                        "url": "https://www.fimfiction.net/tag/twilight-sparkle"
+                    },
+                    {
+                        "id": 234,
+                        "name": "Random",
+                        "old_id": "g:random",
+                        "type": "genre",
+                        "url": "https://www.fimfiction.net/tag/random"
+                    },
+                    {
+                        "id": 120,
+                        "name": "Romance",
+                        "old_id": "g:romance",
+                        "type": "genre",
+                        "url": "https://www.fimfiction.net/tag/romance"
+                    },
+                    {
+                        "id": 4,
+                        "name": "My Little Pony: Friendship is Magic",
+                        "old_id": "",
+                        "type": "series",
+                        "url": "https://www.fimfiction.net/tag/mlp-fim"
+                    }
+                ],
+                "title": "The Greatest Equine Who has Ever Lived!",
+                "total_num_views": 10497,
+                "url": "https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived"
+            }
+        }
+    }
+}

tests/fetchers/test_fimfarchive.py

@@ -5,7 +5,7 @@ Fimfarchive fetcher tests.
 
 #
 # Fimfarchive, preserves stories from Fimfiction.
-# Copyright (C) 2015  Joakim Soderlund
+# Copyright (C) 2019  Joakim Soderlund
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,16 +22,153 @@ Fimfarchive fetcher tests.
 #
 
 
+import json
+from io import BytesIO
+from typing import Any, Dict, List
+from zipfile import ZipFile
+
+import arrow
 import pytest
 
 from fimfarchive.exceptions import InvalidStoryError, StorySourceError
-from fimfarchive.fetchers import FimfarchiveFetcher
+from fimfarchive.fetchers import Fetcher, FimfarchiveFetcher
+from fimfarchive.stories import Story
+from fimfarchive.utils import JayWalker
 
 
 VALID_STORY_KEY = 9
 INVALID_STORY_KEY = 7
 
-FIMFARCHIVE_PATH = 'fimfarchive.zip'
+
+@pytest.fixture(scope='module')
+def data():
+    """
+    Returns test data from JSON.
+    """
+    path = f'{__file__[:-3]}.json'
+
+    with open(path, 'rt') as fobj:
+        return json.load(fobj)
+
+
+class Redactor(JayWalker):
+    """
+    Redacts samples.
+    """
+
+    def handle(self, data, key, value) -> None:
+        if str(key).endswith('_html'):
+            data[key] = '<p>REDACTED<p>'
+        elif key == 'short_description':
+            data[key] = "REDACTED"
+        else:
+            self.walk(value)
+
+
+class FimfarchiveFetcherSampler:
+    """
+    Generates a sample archive for tests.
+
+    Samples must be manually inspected for correctness.
+    """
+
+    def __init__(self, fetcher: Fetcher, *keys: int) -> None:
+        """
+        Constructor.
+
+        Args:
+            fetcher: The fetcher to fetch from.
+            *keys: The stories to sample.
+        """
+        self.redactor = Redactor()
+        self.stories = [self.sample(fetcher, key) for key in keys]
+
+    def sample(self, fetcher: Fetcher, key: int) -> Story:
+        """
+        Returns a redacted story sample.
+        """
+        story = fetcher.fetch(key)
+        story = story.merge(data=b'REDACTED')
+        self.redactor.walk(story.meta)
+
+        return story
+
+    @property
+    def files(self) -> List[Dict[str, str]]:
+        """
+        Returns a list of story data files.
+        """
+        files = []
+
+        for story in self.stories:
+            files.append({
+                'name': story.meta['archive']['path'],
+                'text': story.data.decode(),
+            })
+
+        return files
+
+    @property
+    def about(self) -> Dict[str, str]:
+        """
+        Returns the about file dictionary.
+        """
+        today = arrow.utcnow()
+        fmt = 'YYYYMMDD'
+
+        return {
+            'version': today.shift(days=-1).format(fmt),
+            'start': today.shift(days=-9).format(fmt),
+            'end': today.shift(days=-2).format(fmt),
+        }
+
+    @property
+    def index(self) -> Dict[int, Any]:
+        """
+        Returns the index file dictionary.
+        """
+        return {
+            story.key: story.meta
+            for story in self.stories
+        }
+
+    @property
+    def archive(self) -> Dict[str, Any]:
+        """
+        Returns all the sample content.
+        """
+        return {
+            'about': self.about,
+            'files': self.files,
+            'index': self.index,
+        }
+
+    def __str__(self) -> str:
+        """
+        Serializes all samples.
+        """
+        return json.dumps(
+            {'archive': self.archive},
+            ensure_ascii=False,
+            sort_keys=True,
+            indent=4,
+        )
+
+
+def serialize(obj: Dict) -> str:
+    """
+    Serializes into JSON readable by the fetcher.
+    """
+    entries = []
+
+    for key, value in obj.items():
+        data = json.dumps(value, sort_keys=True)
+        entries.append(f'"{key}": {data}')
+
+    joined = ',\n'.join(entries)
+    output = '\n'.join(('{', joined, '}', ''))
+
+    return output
 
 
 class TestFimfarchiveFetcher:
@@ -40,19 +177,38 @@ class TestFimfarchiveFetcher:
     """
 
     @pytest.fixture(scope='module')
-    def fetcher(self):
+    def archive(self, data):
+        """
+        Returns the archive as a byte stream.
+        """
+        stream = BytesIO()
+
+        zobj = ZipFile(stream, 'w')
+        archive = data['archive']
+
+        for entry in archive['files']:
+            zobj.writestr(entry['name'], entry['text'])
+
+        zobj.writestr('readme.pdf', 'REDACTED')
+        zobj.writestr('about.json', serialize(archive['about']))
+        zobj.writestr('index.json', serialize(archive['index']))
+        zobj.close()
+
+        return stream
+
+    @pytest.fixture()
+    def fetcher(self, archive):
         """
         Returns the fetcher instance to test.
         """
-        with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
+        with FimfarchiveFetcher(archive) as fetcher:
             yield fetcher
 
-    def test_closed_fetcher_raises_exception(self):
+    def test_closed_fetcher_raises_exception(self, fetcher):
         """
         Tests `StorySourceError` is raised when fetcher is closed.
         """
-        with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
+        fetcher.close()
-            fetcher.fetch_meta(VALID_STORY_KEY)
 
         with pytest.raises(StorySourceError):
             fetcher.fetch_meta(VALID_STORY_KEY)
@@ -86,9 +242,9 @@ class TestFimfarchiveFetcher:
             fetcher.fetch_data(INVALID_STORY_KEY)
 
     @pytest.mark.parametrize('attr', ('archive', 'index', 'paths'))
-    def test_close_when_missing_attribute(self, attr):
+    def test_close_when_missing_attribute(self, fetcher, attr):
         """
         Tests close works even after partial initialization.
         """
-        with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
+        delattr(fetcher, attr)
-            delattr(fetcher, attr)
+        fetcher.close()

tox.ini

@@ -17,7 +17,6 @@ commands =
 
 [pytest]
 addopts =
-    --ignore tests/fetchers/test_fimfarchive.py
     tests
 
 [flake8]