PonyMirrors/fimfarchive
1
0
Fork 0
mirror of https://github.com/JockeTF/fimfarchive.git synced 2024-11-22 05:17:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add archive sample for Fimfarchive fetcher tests

Browse source
This commit is contained in:
Joakim Soderlund 2019-03-17 16:09:13 +01:00
parent 1accc8c750
commit 314923a4de
3 changed files with 302 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
tests/fetchers/test_fimfarchive.json Normal file
View file

@ -0,0 +1,135 @@
{
"archive": {
"about": {
"end": "20190315",
"start": "20190308",
"version": "20190316"
},
"files": [
{
"name": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub",
"text": "REDACTED"
}
],
"index": {
"9": {
"archive": {
"date_checked": "2019-01-30T17:25:59.374016+00:00",
"date_created": null,
"date_fetched": "2019-01-30T17:25:59.374016+00:00",
"date_updated": "2017-11-01T21:27:29.364912+00:00",
"path": "epub/s/sethisto-18/the_greatest_equine_who_has_ever_lived-9.epub"
},
"author": {
"avatar": {
"128": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-128",
"160": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-160",
"192": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-192",
"256": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-256",
"32": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-32",
"320": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-320",
"384": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-384",
"48": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-48",
"512": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-512",
"64": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-64",
"96": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-96"
},
"bio_html": "<p>REDACTED<p>",
"date_joined": "2011-06-25T20:53:48+00:00",
"id": 18,
"name": "Sethisto",
"num_blog_posts": 0,
"num_followers": 143,
"num_stories": 1,
"url": "https://www.fimfiction.net/user/18/Sethisto"
},
"chapters": [
{
"chapter_number": 1,
"date_modified": "2014-01-28T11:25:52+00:00",
"date_published": "2011-07-08T18:04:11+00:00",
"id": 10,
"num_views": 10498,
"num_words": 321,
"published": true,
"title": "Chapter 1",
"url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1"
}
],
"color": {
"hex": "3e3e7e",
"rgb": [
62,
62,
126
]
},
"completion_status": "incomplete",
"content_rating": "everyone",
"cover_image": {
"full": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-full",
"large": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-large",
"medium": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-medium",
"thumbnail": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-tiny"
},
"date_modified": null,
"date_published": "2011-07-08T18:04:11+00:00",
"date_updated": "2011-06-25T21:05:53+00:00",
"description_html": "<p>REDACTED<p>",
"id": 9,
"num_chapters": 1,
"num_comments": 228,
"num_dislikes": 52,
"num_likes": 398,
"num_views": 10497,
"num_words": 321,
"prequel": null,
"published": true,
"rating": 88,
"short_description": "REDACTED",
"status": "visible",
"submitted": true,
"tags": [
{
"id": 20,
"name": "Trixie",
"old_id": "c:21",
"type": "character",
"url": "https://www.fimfiction.net/tag/trixie"
},
{
"id": 6,
"name": "Twilight Sparkle",
"old_id": "c:7",
"type": "character",
"url": "https://www.fimfiction.net/tag/twilight-sparkle"
},
{
"id": 234,
"name": "Random",
"old_id": "g:random",
"type": "genre",
"url": "https://www.fimfiction.net/tag/random"
},
{
"id": 120,
"name": "Romance",
"old_id": "g:romance",
"type": "genre",
"url": "https://www.fimfiction.net/tag/romance"
},
{
"id": 4,
"name": "My Little Pony: Friendship is Magic",
"old_id": "",
"type": "series",
"url": "https://www.fimfiction.net/tag/mlp-fim"
}
],
"title": "The Greatest Equine Who has Ever Lived!",
"total_num_views": 10497,
"url": "https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived"
}
}
}
}

176
tests/fetchers/test_fimfarchive.py
View file

@ -5,7 +5,7 @@ Fimfarchive fetcher tests.
#
# Fimfarchive, preserves stories from Fimfiction.
# Copyright (C) 2015 Joakim Soderlund
# Copyright (C) 2019 Joakim Soderlund
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -22,16 +22,153 @@ Fimfarchive fetcher tests.
#
import json
from io import BytesIO
from typing import Any, Dict, List
from zipfile import ZipFile
import arrow
import pytest
from fimfarchive.exceptions import InvalidStoryError, StorySourceError
from fimfarchive.fetchers import FimfarchiveFetcher
from fimfarchive.fetchers import Fetcher, FimfarchiveFetcher
from fimfarchive.stories import Story
from fimfarchive.utils import JayWalker
VALID_STORY_KEY = 9
INVALID_STORY_KEY = 7
FIMFARCHIVE_PATH = 'fimfarchive.zip'
@pytest.fixture(scope='module')
def data():
"""
Returns test data from JSON.
"""
path = f'{__file__[:-3]}.json'
with open(path, 'rt') as fobj:
return json.load(fobj)
class Redactor(JayWalker):
"""
Redacts samples.
"""
def handle(self, data, key, value) -> None:
if str(key).endswith('_html'):
data[key] = '<p>REDACTED<p>'
elif key == 'short_description':
data[key] = "REDACTED"
else:
self.walk(value)
class FimfarchiveFetcherSampler:
"""
Generates a sample archive for tests.
Samples must be manually inspected for correctness.
"""
def __init__(self, fetcher: Fetcher, *keys: int) -> None:
"""
Constructor.
Args:
fetcher: The fetcher to fetch from.
*keys: The stories to sample.
"""
self.redactor = Redactor()
self.stories = [self.sample(fetcher, key) for key in keys]
def sample(self, fetcher: Fetcher, key: int) -> Story:
"""
Returns a redacted story sample.
"""
story = fetcher.fetch(key)
story = story.merge(data=b'REDACTED')
self.redactor.walk(story.meta)
return story
@property
def files(self) -> List[Dict[str, str]]:
"""
Returns a list of story data files.
"""
files = []
for story in self.stories:
files.append({
'name': story.meta['archive']['path'],
'text': story.data.decode(),
})
return files
@property
def about(self) -> Dict[str, str]:
"""
Returns the about file dictionary.
"""
today = arrow.utcnow()
fmt = 'YYYYMMDD'
return {
'version': today.shift(days=-1).format(fmt),
'start': today.shift(days=-9).format(fmt),
'end': today.shift(days=-2).format(fmt),
}
@property
def index(self) -> Dict[int, Any]:
"""
Returns the index file dictionary.
"""
return {
story.key: story.meta
for story in self.stories
}
@property
def archive(self) -> Dict[str, Any]:
"""
Returns all the sample content.
"""
return {
'about': self.about,
'files': self.files,
'index': self.index,
}
def __str__(self) -> str:
"""
Serializes all samples.
"""
return json.dumps(
{'archive': self.archive},
ensure_ascii=False,
sort_keys=True,
indent=4,
)
def serialize(obj: Dict) -> str:
"""
Serializes into JSON readable by the fetcher.
"""
entries = []
for key, value in obj.items():
data = json.dumps(value, sort_keys=True)
entries.append(f'"{key}": {data}')
joined = ',\n'.join(entries)
output = '\n'.join(('{', joined, '}', ''))
return output
class TestFimfarchiveFetcher:
@ -40,19 +177,38 @@ class TestFimfarchiveFetcher:
"""
@pytest.fixture(scope='module')
def fetcher(self):
def archive(self, data):
"""
Returns the archive as a byte stream.
"""
stream = BytesIO()
zobj = ZipFile(stream, 'w')
archive = data['archive']
for entry in archive['files']:
zobj.writestr(entry['name'], entry['text'])
zobj.writestr('readme.pdf', 'REDACTED')
zobj.writestr('about.json', serialize(archive['about']))
zobj.writestr('index.json', serialize(archive['index']))
zobj.close()
return stream
@pytest.fixture()
def fetcher(self, archive):
"""
Returns the fetcher instance to test.
"""
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
with FimfarchiveFetcher(archive) as fetcher:
yield fetcher
def test_closed_fetcher_raises_exception(self):
def test_closed_fetcher_raises_exception(self, fetcher):
"""
Tests `StorySourceError` is raised when fetcher is closed.
"""
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
fetcher.fetch_meta(VALID_STORY_KEY)
fetcher.close()
with pytest.raises(StorySourceError):
fetcher.fetch_meta(VALID_STORY_KEY)
@ -86,9 +242,9 @@ class TestFimfarchiveFetcher:
fetcher.fetch_data(INVALID_STORY_KEY)
@pytest.mark.parametrize('attr', ('archive', 'index', 'paths'))
def test_close_when_missing_attribute(self, attr):
def test_close_when_missing_attribute(self, fetcher, attr):
"""
Tests close works even after partial initialization.
"""
with FimfarchiveFetcher(FIMFARCHIVE_PATH) as fetcher:
delattr(fetcher, attr)
fetcher.close()

1
tox.ini
View file

@ -17,7 +17,6 @@ commands =
[pytest]
addopts =
--ignore tests/fetchers/test_fimfarchive.py
tests
[flake8]