From df4c820ca10bf1a99a4677cf61aede5e47eaf543 Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Wed, 20 Mar 2019 14:08:41 +0100 Subject: [PATCH] Add local timezone to UTC converter --- fimfarchive/converters/__init__.py | 2 + fimfarchive/converters/json_fpub/__init__.py | 41 ++------ fimfarchive/converters/local_utc.py | 71 ++++++++++++++ tests/converters/test_local_utc.py | 98 ++++++++++++++++++++ 4 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 fimfarchive/converters/local_utc.py create mode 100644 tests/converters/test_local_utc.py diff --git a/fimfarchive/converters/__init__.py b/fimfarchive/converters/__init__.py index a072a63..813a406 100644 --- a/fimfarchive/converters/__init__.py +++ b/fimfarchive/converters/__init__.py @@ -25,10 +25,12 @@ Converter module. from .base import Converter from .alpha_beta import AlphaBetaConverter from .json_fpub import JsonFpubConverter +from .local_utc import LocalUtcConverter __all__ = ( 'Converter', 'AlphaBetaConverter', 'JsonFpubConverter', + 'LocalUtcConverter', ) diff --git a/fimfarchive/converters/json_fpub/__init__.py b/fimfarchive/converters/json_fpub/__init__.py index 212ea78..de7cf24 100644 --- a/fimfarchive/converters/json_fpub/__init__.py +++ b/fimfarchive/converters/json_fpub/__init__.py @@ -23,21 +23,19 @@ JSON to FPUB converter for story data. import json -from copy import deepcopy from io import BytesIO -from typing import Any, Dict, Iterator, Optional, Tuple +from typing import Any, Dict, Iterator, Tuple from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED -import arrow from jinja2 import Environment, PackageLoader from fimfarchive.flavors import DataFormat, MetaFormat from fimfarchive.stories import Story -from fimfarchive.utils import JayWalker from fimfarchive.fetchers.fimfiction2 import BetaFormatVerifier from ..base import Converter +from ..local_utc import LocalUtcConverter __all__ = ( @@ -49,29 +47,6 @@ MIMETYPE = 'application/epub+zip' PACKAGE = __package__.rsplit('.', 1) -class DateNormalizer(JayWalker): - """ - Normalizes timezones of date values to UTC. - """ - - def handle(self, data, key, value) -> None: - if str(key).startswith('date_'): - data[key] = self.normalize(value) - else: - self.walk(value) - - def normalize(self, value: Optional[str]) -> Optional[str]: - """ - Normalizes a single date value. - """ - parsed = arrow.get(value or 0) - - if parsed.timestamp == 0: - return None - - return parsed.to('utc').isoformat() - - class StoryRenderer: """ Renders story data. @@ -89,7 +64,6 @@ class StoryRenderer: self.book_opf = env.get_template('book.opf') self.book_ncx = env.get_template('book.ncx') - self.date_normalizer = DateNormalizer() self.verify_meta = BetaFormatVerifier.from_meta_params() self.verify_data = BetaFormatVerifier.from_data_params() @@ -146,7 +120,7 @@ class StoryRenderer: self.verify_index(index, meta['chapter_number']) self.verify_index(index, data['chapter_number']) - yield {**meta, **data} + yield {**data, **meta} def iter_content(self, story: Story) -> Iterator[Tuple[str, str]]: """ @@ -164,11 +138,8 @@ class StoryRenderer: yield path, self.chapter_html.render(chapter) - meta = deepcopy(story.meta) - self.date_normalizer.walk(meta) - - yield 'book.opf', self.book_opf.render(meta) - yield 'book.ncx', self.book_ncx.render(meta) + yield 'book.opf', self.book_opf.render(story.meta) + yield 'book.ncx', self.book_ncx.render(story.meta) def __call__(self, story: Story) -> bytes: """ @@ -192,6 +163,7 @@ class JsonFpubConverter(Converter): def __init__(self) -> None: self.render = StoryRenderer() + self.normalize = LocalUtcConverter() def __call__(self, story: Story) -> Story: if DataFormat.JSON not in story.flavors: @@ -200,6 +172,7 @@ class JsonFpubConverter(Converter): if MetaFormat.BETA not in story.flavors: raise ValueError(f"Missing flavor: {MetaFormat.BETA}") + story = self.normalize(story) data = self.render(story) flavors = set(story.flavors) diff --git a/fimfarchive/converters/local_utc.py b/fimfarchive/converters/local_utc.py new file mode 100644 index 0000000..d1b4eab --- /dev/null +++ b/fimfarchive/converters/local_utc.py @@ -0,0 +1,71 @@ +""" +Local timezone to UTC converter. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2019 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from copy import deepcopy +from typing import Any, Optional + +import arrow + +from fimfarchive.stories import Story +from fimfarchive.utils import JayWalker + +from .base import Converter + + +class DateNormalizer(JayWalker): + """ + Normalizes timezones of date values to UTC. + """ + + def handle(self, data, key, value) -> None: + if str(key).startswith('date_'): + data[key] = self.normalize(value) + else: + self.walk(value) + + def normalize(self, value: Any) -> Optional[str]: + """ + Normalizes a single date value. + """ + parsed = arrow.get(value or 0) + + if parsed.timestamp == 0: + return None + + return parsed.to('utc').isoformat() + + +class LocalUtcConverter(Converter): + """ + Converts date strings to UTC. + """ + + def __init__(self) -> None: + self.normalizer = DateNormalizer() + + def __call__(self, story: Story) -> Story: + meta = deepcopy(story.meta) + self.normalizer.walk(meta) + + return story.merge(meta=meta) diff --git a/tests/converters/test_local_utc.py b/tests/converters/test_local_utc.py new file mode 100644 index 0000000..f057e3b --- /dev/null +++ b/tests/converters/test_local_utc.py @@ -0,0 +1,98 @@ +""" +Local timezone to UTC converter tests. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2019 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +import json +import pytest +from copy import deepcopy + +from fimfarchive.converters import LocalUtcConverter + + +class TestLocalUtcConverter: + """ + LocalUtcConverter tests. + """ + + @pytest.fixture(params=[ + ('2019-03-20T11:27:58+00:00', '2019-03-20T11:27:58+00:00'), + ('2019-03-20T12:29:15+01:00', '2019-03-20T11:29:15+00:00'), + ('1970-01-01T00:00:00+00:00', None), + ('1970-01-01T01:00:00+01:00', None), + (None, None), + ]) + def date_pair(self, request): + """ + Returns a date pair. + """ + local, utc = request.param + + return local, utc + + @pytest.fixture(params=[ + '{"a":{"b":{"c":{"d":"e"}}}}', + '{"a":{"b":{"c":{"date_x":?}}}}', + '{"a":{"b":{"c":{"date_x":?,"date_y":?}}}}', + '{"a":{"b":{"c":{"date_x":?},"date_y":?}}}', + '{"a":{"b":{"c":{"date_x":?}},"date_y":?}}', + '{"date_x":?,"kittens":"2019-03-20T13:06:13+01:00","a":{"date_x":?}}', + '{"date_x":?,"kittens":"2019-03-20T13:06:13+01:00","date_y":?}', + ]) + def meta_pair(self, request, date_pair): + """ + Returns a meta pair. + """ + template = request.param + local_date, utc_date = date_pair + + local_value = json.dumps(local_date) + local_json = template.replace('?', local_value) + local_meta = json.loads(local_json) + + utc_value = json.dumps(utc_date) + utc_json = template.replace('?', utc_value) + utc_meta = json.loads(utc_json) + + return local_meta, utc_meta + + @pytest.fixture + def converter(self): + """ + Returns a converter instance. + """ + return LocalUtcConverter() + + def test_conversion(self, converter, story, meta_pair): + """ + Tests local to UTC conversion. + """ + local_meta, utc_meta = meta_pair + + local_story = story.merge(meta=local_meta) + utc_story = story.merge(meta=utc_meta) + + clone = deepcopy(local_story) + converted = converter(local_story) + + assert clone.meta == local_story.meta + assert utc_story.meta == converted.meta