mirror of
https://github.com/JockeTF/fimfarchive.git
synced 2024-11-22 05:17:59 +01:00
Add JSON to FPUB converter
This commit is contained in:
Joakim Soderlund
parent
82dd078020
commit
eca26dd2c3
10 changed files with 685 additions and 1 deletions
|
|
@ -5,7 +5,7 @@ Converter module.
|
||||||
|
|
||||||
#
|
#
|
||||||
# Fimfarchive, preserves stories from Fimfiction.
|
# Fimfarchive, preserves stories from Fimfiction.
|
||||||
# Copyright (C) 2015 Joakim Soderlund
|
# Copyright (C) 2019 Joakim Soderlund
|
||||||
#
|
#
|
||||||
# This program is free software: you can redistribute it and/or modify
|
# This program is free software: you can redistribute it and/or modify
|
||||||
# it under the terms of the GNU General Public License as published by
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
|
@ -24,9 +24,11 @@ Converter module.
|
||||||
|
|
||||||
from .base import Converter
|
from .base import Converter
|
||||||
from .alpha_beta import AlphaBetaConverter
|
from .alpha_beta import AlphaBetaConverter
|
||||||
|
from .json_fpub import JsonFpubConverter
|
||||||
|
|
||||||
|
|
||||||
__all__ = (
|
__all__ = (
|
||||||
'Converter',
|
'Converter',
|
||||||
'AlphaBetaConverter',
|
'AlphaBetaConverter',
|
||||||
|
'JsonFpubConverter',
|
||||||
)
|
)
|
||||||
|
|
|
||||||
209
fimfarchive/converters/json_fpub/__init__.py
Normal file
209
fimfarchive/converters/json_fpub/__init__.py
Normal file
|
|
@ -0,0 +1,209 @@
|
||||||
|
"""
|
||||||
|
JSON to FPUB converter for story data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# Fimfarchive, preserves stories from Fimfiction.
|
||||||
|
# Copyright (C) 2019 Joakim Soderlund
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
import json
|
||||||
|
from copy import deepcopy
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import Any, Dict, Iterator, Optional, Tuple
|
||||||
|
from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
|
||||||
|
|
||||||
|
import arrow
|
||||||
|
from jinja2 import Environment, PackageLoader
|
||||||
|
|
||||||
|
from fimfarchive.flavors import DataFormat, MetaFormat
|
||||||
|
from fimfarchive.stories import Story
|
||||||
|
from fimfarchive.utils import JayWalker
|
||||||
|
|
||||||
|
from fimfarchive.fetchers.fimfiction2 import BetaFormatVerifier
|
||||||
|
|
||||||
|
from ..base import Converter
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
'JsonFpubConverter',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
MIMETYPE = 'application/epub+zip'
|
||||||
|
PACKAGE = __package__.rsplit('.', 1)
|
||||||
|
|
||||||
|
|
||||||
|
class DateNormalizer(JayWalker):
|
||||||
|
"""
|
||||||
|
Normalizes timezones of date values to UTC.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def handle(self, data, key, value) -> None:
|
||||||
|
if str(key).startswith('date_'):
|
||||||
|
data[key] = self.normalize(value)
|
||||||
|
else:
|
||||||
|
self.walk(value)
|
||||||
|
|
||||||
|
def normalize(self, value: Optional[str]) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Normalizes a single date value.
|
||||||
|
"""
|
||||||
|
parsed = arrow.get(value or 0)
|
||||||
|
|
||||||
|
if parsed.timestamp == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return parsed.to('utc').isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
class StoryRenderer:
|
||||||
|
"""
|
||||||
|
Renders story data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
env = Environment(
|
||||||
|
autoescape=True,
|
||||||
|
keep_trailing_newline=True,
|
||||||
|
loader=PackageLoader(*PACKAGE),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.container_xml = env.get_template('container.xml')
|
||||||
|
self.chapter_html = env.get_template('chapter.html')
|
||||||
|
self.book_opf = env.get_template('book.opf')
|
||||||
|
self.book_ncx = env.get_template('book.ncx')
|
||||||
|
|
||||||
|
self.date_normalizer = DateNormalizer()
|
||||||
|
self.verify_meta = BetaFormatVerifier.from_meta_params()
|
||||||
|
self.verify_data = BetaFormatVerifier.from_data_params()
|
||||||
|
|
||||||
|
def fix_authors_note_position(self, data: Dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Clears author's note position if author's note is missing.
|
||||||
|
"""
|
||||||
|
authors_note = data['authors_note_html']
|
||||||
|
|
||||||
|
if not authors_note or len(authors_note.strip()) < 10:
|
||||||
|
data['authors_note_position'] = None
|
||||||
|
|
||||||
|
def fix_local_href_attributes(self, data: Dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Replaces local href attributes with global ones.
|
||||||
|
"""
|
||||||
|
source = ' href="/'
|
||||||
|
target = ' href="https://www.fimfiction.net/'
|
||||||
|
|
||||||
|
for key in ('authors_note_html', 'content_html'):
|
||||||
|
data[key] = data[key].replace(source, target)
|
||||||
|
|
||||||
|
def fix_local_src_attributes(self, data: Dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Replaces local src attributes with global ones.
|
||||||
|
"""
|
||||||
|
source = ' src="/'
|
||||||
|
target = ' src="https://www.fimfiction.net/'
|
||||||
|
|
||||||
|
for key in ('authors_note_html', 'content_html'):
|
||||||
|
data[key] = data[key].replace(source, target)
|
||||||
|
|
||||||
|
def verify_index(self, expected, actual):
|
||||||
|
"""
|
||||||
|
Raises an exception if the index values differ.
|
||||||
|
"""
|
||||||
|
if expected != actual:
|
||||||
|
raise ValueError(f"Expected index {expected}, was {actual}.")
|
||||||
|
|
||||||
|
def iter_chapters(self, story: Story) -> Iterator[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Yields chapter meta and data, combined into one.
|
||||||
|
"""
|
||||||
|
self.verify_meta(story.meta)
|
||||||
|
|
||||||
|
meta_chapters = story.meta['chapters']
|
||||||
|
data_chapters = json.loads(story.data.decode())
|
||||||
|
zipped = zip(meta_chapters, data_chapters)
|
||||||
|
|
||||||
|
for index, chapter in enumerate(zipped, 1):
|
||||||
|
meta, data = chapter
|
||||||
|
|
||||||
|
self.verify_data(data)
|
||||||
|
self.verify_index(index, meta['chapter_number'])
|
||||||
|
self.verify_index(index, data['chapter_number'])
|
||||||
|
|
||||||
|
yield {**meta, **data}
|
||||||
|
|
||||||
|
def iter_content(self, story: Story) -> Iterator[Tuple[str, str]]:
|
||||||
|
"""
|
||||||
|
Yields EPUB file paths and content for a story.
|
||||||
|
"""
|
||||||
|
yield 'META-INF/container.xml', self.container_xml.render()
|
||||||
|
|
||||||
|
for chapter in self.iter_chapters(story):
|
||||||
|
number = chapter['chapter_number']
|
||||||
|
path = f"Chapter{number}.html"
|
||||||
|
|
||||||
|
self.fix_authors_note_position(chapter)
|
||||||
|
self.fix_local_href_attributes(chapter)
|
||||||
|
self.fix_local_src_attributes(chapter)
|
||||||
|
|
||||||
|
yield path, self.chapter_html.render(chapter)
|
||||||
|
|
||||||
|
meta = deepcopy(story.meta)
|
||||||
|
self.date_normalizer.walk(meta)
|
||||||
|
|
||||||
|
yield 'book.opf', self.book_opf.render(meta)
|
||||||
|
yield 'book.ncx', self.book_ncx.render(meta)
|
||||||
|
|
||||||
|
def __call__(self, story: Story) -> bytes:
|
||||||
|
"""
|
||||||
|
Renders the EPUB file contents as bytes.
|
||||||
|
"""
|
||||||
|
fobj = BytesIO()
|
||||||
|
|
||||||
|
with ZipFile(fobj, 'w') as archive:
|
||||||
|
archive.writestr('mimetype', MIMETYPE, ZIP_STORED)
|
||||||
|
|
||||||
|
for path, data in self.iter_content(story):
|
||||||
|
archive.writestr(path, data, ZIP_DEFLATED)
|
||||||
|
|
||||||
|
return fobj.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFpubConverter(Converter):
|
||||||
|
"""
|
||||||
|
Converts story data from JSON to FPUB format.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.render = StoryRenderer()
|
||||||
|
|
||||||
|
def __call__(self, story: Story) -> Story:
|
||||||
|
if DataFormat.JSON not in story.flavors:
|
||||||
|
raise ValueError(f"Missing flavor: {DataFormat.JSON}")
|
||||||
|
|
||||||
|
if MetaFormat.BETA not in story.flavors:
|
||||||
|
raise ValueError(f"Missing flavor: {MetaFormat.BETA}")
|
||||||
|
|
||||||
|
data = self.render(story)
|
||||||
|
|
||||||
|
flavors = set(story.flavors)
|
||||||
|
flavors.remove(DataFormat.JSON)
|
||||||
|
flavors.add(DataFormat.FPUB)
|
||||||
|
|
||||||
|
return story.merge(data=data, flavors=flavors)
|
||||||
28
fimfarchive/converters/json_fpub/book.ncx
Normal file
28
fimfarchive/converters/json_fpub/book.ncx
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
|
||||||
|
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1" xml:lang="en">
|
||||||
|
<head>
|
||||||
|
<meta name="dtb:uid" content="{{ url }}"/>
|
||||||
|
<meta name="dtb:depth" content="2"/>
|
||||||
|
<meta name="dtb:totalPageCount" content="0"/>
|
||||||
|
<meta name="dtb:maxPageNumber" content="0"/>
|
||||||
|
<meta name="dtb:generator" content="Fimfarchive"/>
|
||||||
|
</head>
|
||||||
|
<docTitle>
|
||||||
|
<text>{{ title }}</text>
|
||||||
|
</docTitle>
|
||||||
|
<docAuthor>
|
||||||
|
<text>{{ author.name }}</text>
|
||||||
|
</docAuthor>
|
||||||
|
<navMap>
|
||||||
|
{%- for chapter in chapters %}
|
||||||
|
{%- set cnum = chapter.chapter_number %}
|
||||||
|
<navPoint id="c{{ cnum }}" playOrder="{{ cnum }}">
|
||||||
|
<navLabel>
|
||||||
|
<text>{{ chapter.title }}</text>
|
||||||
|
</navLabel>
|
||||||
|
<content src="Chapter{{ cnum }}.html"/>
|
||||||
|
</navPoint>
|
||||||
|
{%- endfor %}
|
||||||
|
</navMap>
|
||||||
|
</ncx>
|
||||||
30
fimfarchive/converters/json_fpub/book.opf
Normal file
30
fimfarchive/converters/json_fpub/book.opf
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="BookId" version="2.0">
|
||||||
|
<metadata
|
||||||
|
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xmlns:opf="http://www.idpf.org/2007/opf"
|
||||||
|
xmlns:dcterms="http://purl.org/dc/terms/">
|
||||||
|
<dc:title>{{ title }}</dc:title>
|
||||||
|
<dc:language>en</dc:language>
|
||||||
|
<dc:identifier id="BookId" opf:scheme="URI">{{ url }}</dc:identifier>
|
||||||
|
<dc:description>{{ short_description }}</dc:description>
|
||||||
|
<dc:publisher>Fimfiction</dc:publisher>
|
||||||
|
<dc:relation>http://www.fimfiction.net</dc:relation>
|
||||||
|
<dc:creator opf:file-as="{{ author.name }}" opf:role="aut">{{ author.name }}</dc:creator>
|
||||||
|
<dc:date>{{ date_updated }}</dc:date>
|
||||||
|
<dc:source>{{ url }}</dc:source>
|
||||||
|
</metadata>
|
||||||
|
<manifest>
|
||||||
|
<item id="ncx" href="book.ncx" media-type="application/x-dtbncx+xml"/>
|
||||||
|
{%- for chapter in chapters %}
|
||||||
|
{%- set cnum = chapter.chapter_number %}
|
||||||
|
<item id="c{{ cnum }}" href="Chapter{{ cnum }}.html" media-type="application/xhtml+xml"/>
|
||||||
|
{%- endfor %}
|
||||||
|
</manifest>
|
||||||
|
<spine toc="ncx">
|
||||||
|
{%- for chapter in chapters %}
|
||||||
|
<itemref idref="c{{ chapter.chapter_number }}"/>
|
||||||
|
{%- endfor %}
|
||||||
|
</spine>
|
||||||
|
</package>
|
||||||
24
fimfarchive/converters/json_fpub/chapter.html
Normal file
24
fimfarchive/converters/json_fpub/chapter.html
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
|
||||||
|
<title>{{ title }}</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
{%- autoescape false %}
|
||||||
|
{%- if authors_note_position == 'top' %}
|
||||||
|
<h1>{{ "Author's Note" | e }}</h1>
|
||||||
|
<div id='authors-note'>{{ authors_note_html }}</div>
|
||||||
|
<hr/>
|
||||||
|
{%- endif %}
|
||||||
|
<h1>{{ title | e }}</h1>
|
||||||
|
<div id='content'>{{ content_html }}</div>
|
||||||
|
{%- if authors_note_position == 'bottom' %}
|
||||||
|
<hr/>
|
||||||
|
<h1>{{ "Author's Note" | e }}</h1>
|
||||||
|
<div id='authors-note'>{{ authors_note_html }}</div>
|
||||||
|
{%- endif %}
|
||||||
|
{%- endautoescape %}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
6
fimfarchive/converters/json_fpub/container.xml
Normal file
6
fimfarchive/converters/json_fpub/container.xml
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||||
|
<rootfiles>
|
||||||
|
<rootfile full-path="book.opf" media-type="application/oebps-package+xml"/>
|
||||||
|
</rootfiles>
|
||||||
|
</container>
|
||||||
|
|
@ -4,6 +4,7 @@ blinker
|
||||||
flake8
|
flake8
|
||||||
jsonapi-client
|
jsonapi-client
|
||||||
importlib_resources
|
importlib_resources
|
||||||
|
jinja2
|
||||||
jmespath
|
jmespath
|
||||||
mypy
|
mypy
|
||||||
pytest
|
pytest
|
||||||
|
|
|
||||||
1
setup.py
1
setup.py
|
|
@ -89,6 +89,7 @@ setup(
|
||||||
'bbcode',
|
'bbcode',
|
||||||
'blinker',
|
'blinker',
|
||||||
'importlib_resources',
|
'importlib_resources',
|
||||||
|
'jinja2',
|
||||||
'jmespath',
|
'jmespath',
|
||||||
'jsonapi-client',
|
'jsonapi-client',
|
||||||
'requests',
|
'requests',
|
||||||
|
|
|
||||||
151
tests/converters/test_json_fpub.json
Normal file
151
tests/converters/test_json_fpub.json
Normal file
|
|
@ -0,0 +1,151 @@
|
||||||
|
{
|
||||||
|
"pairs": [
|
||||||
|
{
|
||||||
|
"fpub": [
|
||||||
|
{
|
||||||
|
"name": "mimetype",
|
||||||
|
"text": "application/epub+zip"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "META-INF/container.xml",
|
||||||
|
"text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<container version=\"1.0\" xmlns=\"urn:oasis:names:tc:opendocument:xmlns:container\">\n <rootfiles>\n <rootfile full-path=\"book.opf\" media-type=\"application/oebps-package+xml\"/>\n </rootfiles>\n</container>\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Chapter1.html",
|
||||||
|
"text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\n <head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>\n <title>Chapter 1</title>\n </head>\n <body>\n <h1>Chapter 1</h1>\n <div id='content'><p>REDACTED<p></div>\n <hr/>\n <h1>Author's Note</h1>\n <div id='authors-note'><p>REDACTED<p></div>\n </body>\n</html>\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "book.opf",
|
||||||
|
"text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<package xmlns=\"http://www.idpf.org/2007/opf\" unique-identifier=\"BookId\" version=\"2.0\">\n <metadata\n xmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xmlns:opf=\"http://www.idpf.org/2007/opf\"\n xmlns:dcterms=\"http://purl.org/dc/terms/\">\n <dc:title>The Greatest Equine Who has Ever Lived!</dc:title>\n <dc:language>en</dc:language>\n <dc:identifier id=\"BookId\" opf:scheme=\"URI\">https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived</dc:identifier>\n <dc:description>REDACTED</dc:description>\n <dc:publisher>Fimfiction</dc:publisher>\n <dc:relation>http://www.fimfiction.net</dc:relation>\n <dc:creator opf:file-as=\"Sethisto\" opf:role=\"aut\">Sethisto</dc:creator>\n <dc:date>2011-06-25T21:05:53+00:00</dc:date>\n <dc:source>https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived</dc:source>\n </metadata>\n <manifest>\n <item id=\"ncx\" href=\"book.ncx\" media-type=\"application/x-dtbncx+xml\"/>\n <item id=\"c1\" href=\"Chapter1.html\" media-type=\"application/xhtml+xml\"/>\n </manifest>\n <spine toc=\"ncx\">\n <itemref idref=\"c1\"/>\n </spine>\n</package>\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "book.ncx",
|
||||||
|
"text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE ncx PUBLIC \"-//NISO//DTD ncx 2005-1//EN\" \"http://www.daisy.org/z3986/2005/ncx-2005-1.dtd\">\n<ncx xmlns=\"http://www.daisy.org/z3986/2005/ncx/\" version=\"2005-1\" xml:lang=\"en\">\n <head>\n <meta name=\"dtb:uid\" content=\"https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived\"/>\n <meta name=\"dtb:depth\" content=\"2\"/>\n <meta name=\"dtb:totalPageCount\" content=\"0\"/>\n <meta name=\"dtb:maxPageNumber\" content=\"0\"/>\n <meta name=\"dtb:generator\" content=\"Fimfarchive\"/>\n </head>\n <docTitle>\n <text>The Greatest Equine Who has Ever Lived!</text>\n </docTitle>\n <docAuthor>\n <text>Sethisto</text>\n </docAuthor>\n <navMap>\n <navPoint id=\"c1\" playOrder=\"1\">\n <navLabel>\n <text>Chapter 1</text>\n </navLabel>\n <content src=\"Chapter1.html\"/>\n </navPoint>\n </navMap>\n</ncx>\n"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"json": [
|
||||||
|
{
|
||||||
|
"authors_note_html": "<p>REDACTED<p>",
|
||||||
|
"authors_note_position": "bottom",
|
||||||
|
"chapter_number": 1,
|
||||||
|
"content_html": "<p>REDACTED<p>",
|
||||||
|
"id": 10,
|
||||||
|
"title": "Chapter 1",
|
||||||
|
"url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"key": 9,
|
||||||
|
"meta": {
|
||||||
|
"author": {
|
||||||
|
"avatar": {
|
||||||
|
"128": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-128",
|
||||||
|
"160": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-160",
|
||||||
|
"192": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-192",
|
||||||
|
"256": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-256",
|
||||||
|
"32": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-32",
|
||||||
|
"320": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-320",
|
||||||
|
"384": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-384",
|
||||||
|
"48": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-48",
|
||||||
|
"512": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-512",
|
||||||
|
"64": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-64",
|
||||||
|
"96": "https://cdn-img.fimfiction.net/user/t74v-1431818459-18-96"
|
||||||
|
},
|
||||||
|
"bio_html": "<p>REDACTED<p>",
|
||||||
|
"date_joined": "2011-06-25T16:53:48-04:00",
|
||||||
|
"id": 18,
|
||||||
|
"name": "Sethisto",
|
||||||
|
"num_blog_posts": 0,
|
||||||
|
"num_followers": 146,
|
||||||
|
"num_stories": 1,
|
||||||
|
"url": "https://www.fimfiction.net/user/18/Sethisto"
|
||||||
|
},
|
||||||
|
"chapters": [
|
||||||
|
{
|
||||||
|
"chapter_number": 1,
|
||||||
|
"date_modified": "2014-01-28T06:25:52-05:00",
|
||||||
|
"date_published": "2011-07-08T14:04:11-04:00",
|
||||||
|
"id": 10,
|
||||||
|
"num_views": 10549,
|
||||||
|
"num_words": 321,
|
||||||
|
"published": true,
|
||||||
|
"title": "Chapter 1",
|
||||||
|
"url": "https://www.fimfiction.net/story/9/1/the-greatest-equine-who-has-ever-lived/chapter-1"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"color": {
|
||||||
|
"hex": "3e3e7e",
|
||||||
|
"rgb": [
|
||||||
|
62,
|
||||||
|
62,
|
||||||
|
126
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"completion_status": "incomplete",
|
||||||
|
"content_rating": "everyone",
|
||||||
|
"cover_image": {
|
||||||
|
"full": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-full",
|
||||||
|
"large": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-large",
|
||||||
|
"medium": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-medium",
|
||||||
|
"thumbnail": "https://cdn-img.fimfiction.net/story/vr3n-1432418803-9-tiny"
|
||||||
|
},
|
||||||
|
"date_modified": "1970-01-01T00:00:00+00:00",
|
||||||
|
"date_published": "2011-07-08T18:04:11+00:00",
|
||||||
|
"date_updated": "2011-06-25T21:05:53+00:00",
|
||||||
|
"description_html": "<p>REDACTED<p>",
|
||||||
|
"id": 9,
|
||||||
|
"num_chapters": 1,
|
||||||
|
"num_comments": 228,
|
||||||
|
"num_dislikes": 52,
|
||||||
|
"num_likes": 399,
|
||||||
|
"num_views": 10549,
|
||||||
|
"num_words": 321,
|
||||||
|
"prequel": null,
|
||||||
|
"published": true,
|
||||||
|
"rating": 88,
|
||||||
|
"short_description": "REDACTED",
|
||||||
|
"status": "visible",
|
||||||
|
"submitted": true,
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"id": 20,
|
||||||
|
"name": "Trixie",
|
||||||
|
"old_id": "c:21",
|
||||||
|
"type": "character",
|
||||||
|
"url": "https://www.fimfiction.net/tag/trixie"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"name": "Twilight Sparkle",
|
||||||
|
"old_id": "c:7",
|
||||||
|
"type": "character",
|
||||||
|
"url": "https://www.fimfiction.net/tag/twilight-sparkle"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 234,
|
||||||
|
"name": "Random",
|
||||||
|
"old_id": "g:random",
|
||||||
|
"type": "genre",
|
||||||
|
"url": "https://www.fimfiction.net/tag/random"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 120,
|
||||||
|
"name": "Romance",
|
||||||
|
"old_id": "g:romance",
|
||||||
|
"type": "genre",
|
||||||
|
"url": "https://www.fimfiction.net/tag/romance"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"name": "My Little Pony: Friendship is Magic",
|
||||||
|
"old_id": "",
|
||||||
|
"type": "series",
|
||||||
|
"url": "https://www.fimfiction.net/tag/mlp-fim"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "The Greatest Equine Who has Ever Lived!",
|
||||||
|
"total_num_views": 10549,
|
||||||
|
"url": "https://www.fimfiction.net/story/9/the-greatest-equine-who-has-ever-lived"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
232
tests/converters/test_json_fpub.py
Normal file
232
tests/converters/test_json_fpub.py
Normal file
|
|
@ -0,0 +1,232 @@
|
||||||
|
"""
|
||||||
|
JSON to FPUB converter tests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# Fimfarchive, preserves stories from Fimfiction.
|
||||||
|
# Copyright (C) 2019 Joakim Soderlund
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
import json
|
||||||
|
from copy import deepcopy
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import Any, Dict, Iterator, List
|
||||||
|
from zipfile import ZipFile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from fimfarchive.converters import JsonFpubConverter
|
||||||
|
from fimfarchive.fetchers import Fimfiction2Fetcher
|
||||||
|
from fimfarchive.flavors import DataFormat, MetaFormat
|
||||||
|
from fimfarchive.stories import Story
|
||||||
|
from fimfarchive.utils import JayWalker
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='module')
|
||||||
|
def data():
|
||||||
|
"""
|
||||||
|
Returns test data from JSON.
|
||||||
|
"""
|
||||||
|
path = f'{__file__[:-3]}.json'
|
||||||
|
|
||||||
|
with open(path, 'rt') as fobj:
|
||||||
|
return json.load(fobj)
|
||||||
|
|
||||||
|
|
||||||
|
class Redactor(JayWalker):
|
||||||
|
"""
|
||||||
|
Redacts samples.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def handle(self, data, key, value) -> None:
|
||||||
|
if str(key).endswith('_html'):
|
||||||
|
data[key] = '<p>REDACTED<p>'
|
||||||
|
elif key == 'short_description':
|
||||||
|
data[key] = "REDACTED"
|
||||||
|
else:
|
||||||
|
self.walk(value)
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFpubConverterSampler:
|
||||||
|
"""
|
||||||
|
Generates sample conversions for tests.
|
||||||
|
|
||||||
|
Samples must be manually inspected for correctness.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, token: str, *keys: int) -> None:
|
||||||
|
"""
|
||||||
|
Constructor.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
token: Fimfiction APIv2 access token.
|
||||||
|
keys: Stories to generate samples for.
|
||||||
|
"""
|
||||||
|
self.keys = sorted(int(key) for key in keys)
|
||||||
|
self.fetcher = Fimfiction2Fetcher(token)
|
||||||
|
self.convert = JsonFpubConverter()
|
||||||
|
self.redactor = Redactor()
|
||||||
|
|
||||||
|
def sample(self, key: int) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Generates a sample conversion.
|
||||||
|
"""
|
||||||
|
story = self.fetcher.fetch(key)
|
||||||
|
redacted = self.redact(story)
|
||||||
|
converted = self.convert(redacted)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'key': int(key),
|
||||||
|
'meta': redacted.meta,
|
||||||
|
'json': json.loads(redacted.data.decode()),
|
||||||
|
'fpub': self.extract(converted.data),
|
||||||
|
}
|
||||||
|
|
||||||
|
def redact(self, story: Story) -> Story:
|
||||||
|
"""
|
||||||
|
Redacts a story.
|
||||||
|
"""
|
||||||
|
meta = deepcopy(story.meta)
|
||||||
|
data = json.loads(story.data.decode())
|
||||||
|
|
||||||
|
self.redactor.walk(meta)
|
||||||
|
self.redactor.walk(data)
|
||||||
|
|
||||||
|
raw_data = json.dumps(data).encode()
|
||||||
|
|
||||||
|
return story.merge(meta=meta, data=raw_data)
|
||||||
|
|
||||||
|
def extract(self, data: bytes) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Lists the contents of a ZIP-file.
|
||||||
|
"""
|
||||||
|
output: List[Dict[str, Any]] = []
|
||||||
|
zobj = ZipFile(BytesIO(data))
|
||||||
|
|
||||||
|
for info in zobj.infolist():
|
||||||
|
output.append({
|
||||||
|
'name': info.filename,
|
||||||
|
'text': zobj.read(info).decode(),
|
||||||
|
})
|
||||||
|
|
||||||
|
return output
|
||||||
|
|
||||||
|
def __iter__(self) -> Iterator[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Yields all samples.
|
||||||
|
"""
|
||||||
|
for key in self.keys:
|
||||||
|
yield self.sample(key)
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
"""
|
||||||
|
Serializes all samples.
|
||||||
|
"""
|
||||||
|
return json.dumps(
|
||||||
|
obj={'pairs': list(self)},
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
indent=4,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestJsonFpubConverter:
|
||||||
|
"""
|
||||||
|
JsonFpubConverter tests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def converter(self):
|
||||||
|
"""
|
||||||
|
Returns a JSON to FPUB converter instance.
|
||||||
|
"""
|
||||||
|
return JsonFpubConverter()
|
||||||
|
|
||||||
|
@pytest.fixture(params=range(1))
|
||||||
|
def pair(self, request, data):
|
||||||
|
"""
|
||||||
|
Returns test data pairs.
|
||||||
|
"""
|
||||||
|
return data['pairs'][request.param]
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def json_story(self, pair):
|
||||||
|
"""
|
||||||
|
Returns a story in the JSON data format.
|
||||||
|
"""
|
||||||
|
return Story(
|
||||||
|
key=pair['key'],
|
||||||
|
meta=deepcopy(pair['meta']),
|
||||||
|
data=json.dumps(pair['json']).encode(),
|
||||||
|
flavors={MetaFormat.BETA, DataFormat.JSON},
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fpub_story(self, pair):
|
||||||
|
"""
|
||||||
|
Returns a story in the FPUB data format.
|
||||||
|
"""
|
||||||
|
stream = BytesIO()
|
||||||
|
|
||||||
|
with ZipFile(stream, 'w') as zobj:
|
||||||
|
for info in pair['fpub']:
|
||||||
|
zobj.writestr(info['name'], info['text'])
|
||||||
|
|
||||||
|
return Story(
|
||||||
|
key=pair['key'],
|
||||||
|
meta=deepcopy(pair['meta']),
|
||||||
|
data=stream.getvalue(),
|
||||||
|
flavors={MetaFormat.BETA, DataFormat.FPUB},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_conversion(self, converter, json_story, fpub_story):
|
||||||
|
"""
|
||||||
|
Tests conversion of story data from JSON to FPUB format.
|
||||||
|
"""
|
||||||
|
converted = converter(json_story)
|
||||||
|
|
||||||
|
exp = ZipFile(BytesIO(fpub_story.data))
|
||||||
|
act = ZipFile(BytesIO(converted.data))
|
||||||
|
|
||||||
|
for einfo, ainfo in zip(exp.infolist(), act.infolist()):
|
||||||
|
assert einfo.filename == ainfo.filename
|
||||||
|
assert exp.read(einfo) == act.read(ainfo)
|
||||||
|
|
||||||
|
def test_mimetype(self, converter, json_story):
|
||||||
|
"""
|
||||||
|
Tests mimetype is included correctly.
|
||||||
|
"""
|
||||||
|
converted = converter(json_story)
|
||||||
|
|
||||||
|
zobj = ZipFile(BytesIO(converted.data))
|
||||||
|
info = zobj.infolist()[0]
|
||||||
|
read = zobj.read(info).decode()
|
||||||
|
|
||||||
|
assert 0 == info.compress_type
|
||||||
|
assert 'mimetype' == info.filename
|
||||||
|
assert 'application/epub+zip' == read
|
||||||
|
|
||||||
|
def test_immutablilty(self, converter, json_story):
|
||||||
|
"""
|
||||||
|
Tests converter doesn't modify original.
|
||||||
|
"""
|
||||||
|
clone = deepcopy(json_story)
|
||||||
|
converter(json_story)
|
||||||
|
|
||||||
|
for attr in ('key', 'fetcher', 'meta', 'data', 'flavors'):
|
||||||
|
assert getattr(clone, attr) == getattr(json_story, attr)
|
||||||
Loading…
Reference in a new issue