From d05b6fd070c96a82f536c85fc576324802b6e645 Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Sun, 29 Mar 2020 17:52:26 +0200 Subject: [PATCH] Add build task --- fimfarchive/tasks/__init__.py | 4 +- fimfarchive/tasks/build.py | 156 ++++++++++++++++++++++++++++ tests/tasks/conftest.py | 67 ++++++++++-- tests/tasks/test_build.py | 185 ++++++++++++++++++++++++++++++++++ 4 files changed, 401 insertions(+), 11 deletions(-) create mode 100644 fimfarchive/tasks/build.py create mode 100644 tests/tasks/test_build.py diff --git a/fimfarchive/tasks/__init__.py b/fimfarchive/tasks/__init__.py index 61e4348..b2bb66a 100644 --- a/fimfarchive/tasks/__init__.py +++ b/fimfarchive/tasks/__init__.py @@ -5,7 +5,7 @@ Tasks module. # # Fimfarchive, preserves stories from Fimfiction. -# Copyright (C) 2015 Joakim Soderlund +# Copyright (C) 2020 Joakim Soderlund # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,9 +22,11 @@ Tasks module. # +from .build import BuildTask from .update import UpdateTask __all__ = ( + 'BuildTask', 'UpdateTask', ) diff --git a/fimfarchive/tasks/build.py b/fimfarchive/tasks/build.py new file mode 100644 index 0000000..5d09883 --- /dev/null +++ b/fimfarchive/tasks/build.py @@ -0,0 +1,156 @@ +""" +Build task. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2020 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from pathlib import Path +from typing import Iterable, Iterator, Optional, Tuple, Union + +import arrow + +from fimfarchive.converters import LocalUtcConverter +from fimfarchive.exceptions import InvalidStoryError, StorySourceError +from fimfarchive.fetchers import Fetcher +from fimfarchive.stories import Story +from fimfarchive.utils import is_blacklisted +from fimfarchive.writers import FimfarchiveWriter + + +__all__ = ( + 'BuildTask', +) + + +PathArg = Union[Path, str] + + +class BuildTask: + """ + Build a new version of the archive. + """ + + def __init__( + self, + output: PathArg, + upcoming: Iterable[Story], + previous: Fetcher = None, + extras: PathArg = None, + ) -> None: + """ + Constructor. + + Args: + output: Directory for the output + upcoming: Upcoming archive content + previous: Previous archive content + extras: Directory for extra files + """ + self.previous = previous + self.upcoming = upcoming + + self.convert = LocalUtcConverter() + self.output = self.get_output(output) + self.extras = self.get_extras(extras) + + def get_output(self, directory: PathArg) -> Path: + """ + Returns the complete path for the new archive. + + Args: + directory: Path for the new archive + """ + date = arrow.utcnow().format('YYYYMMDD') + path = Path(directory).resolve(True) + name = f'fimfarchive-{date}.zip' + + return path / name + + def get_extras( + self, + directory: Optional[PathArg] + ) -> Iterator[Tuple[str, bytes]]: + """ + Yields extra archive data. + + Args: + directory: Path to read extras from + """ + if directory is None: + return () + + for path in Path(directory).iterdir(): + yield path.name, path.read_bytes() + + def revive(self, story: Story) -> Story: + """ + Returns a story containing data from the previous archive. + + Args: + story: The story to revive + + Raises: + StorySourceError: If story data is missing + """ + if self.previous is None: + raise StorySourceError("Missing previous fetcher.") + + try: + revived = self.previous.fetch(story.key) + except InvalidStoryError as e: + raise StorySourceError("Missing revived story.") from e + else: + return story.merge(data=revived.data) + + def resolve(self, story: Story) -> Story: + """ + Returns a story guaranteed to contain data. + + Args: + strory: The story to resolve + + Raises: + StorySourceError: If story data is missing + """ + try: + story.data + except InvalidStoryError: + return self.revive(story) + else: + return story + + def generate(self) -> Iterator[Story]: + """ + Yields stories for the new archive. + """ + for story in self.upcoming: + if is_blacklisted(story): + continue + + converted = self.convert(story) + resolved = self.resolve(converted) + + yield resolved + + def run(self): + with FimfarchiveWriter(self.output, self.extras) as writer: + for story in self.generate(): + writer.write(story) diff --git a/tests/tasks/conftest.py b/tests/tasks/conftest.py index 918e9ed..04afc92 100644 --- a/tests/tasks/conftest.py +++ b/tests/tasks/conftest.py @@ -22,11 +22,26 @@ Common task fixtures. # +from copy import deepcopy from typing import Dict from fimfarchive.exceptions import InvalidStoryError +from fimfarchive.converters import Converter from fimfarchive.fetchers import Fetcher from fimfarchive.stories import Story +from fimfarchive.utils import Empty + + +class DummyConverer(Converter): + """ + Converter that increments a counter. + """ + + def __call__(self, story: Story) -> Story: + meta = deepcopy(story.meta) + meta['conversions'] += 1 + + return story.merge(meta=meta) class DummyFetcher(Fetcher): @@ -40,28 +55,41 @@ class DummyFetcher(Fetcher): """ self.stories: Dict[int, Story] = dict() - def add(self, key, date, flavors=()): + def add(self, key, date, flavors=(), data=Empty): """ Adds a story to the fetcher. """ + meta = { + 'id': key, + 'title': f't{key}', + 'date_modified': date, + 'conversions': 0, + 'author': { + 'id': key, + 'name': f'n{key}' + }, + 'chapters': [ + {'id': key}, + ], + } + + if data is Empty: + text = f'd{key}' + data = text.encode() + story = Story( key=key, + fetcher=self, + meta=meta, + data=data, flavors=flavors, - data=f'Story {key}'.encode(), - meta={ - 'id': key, - 'date_modified': date, - 'chapters': [ - {'id': key}, - ], - }, ) self.stories[key] = story return story - def fetch(self, key): + def fetch(self, key, prefetch_meta=None, prefetch_data=None): """ Returns a previously stored story. """ @@ -69,3 +97,22 @@ class DummyFetcher(Fetcher): return self.stories[key] except KeyError: raise InvalidStoryError() + + def fetch_data(self, key): + """ + Raises exception for missing data. + """ + raise InvalidStoryError() + + def fetch_meta(self, key): + """ + Raises exception for missing meta. + """ + raise InvalidStoryError() + + def __iter__(self): + """ + Yields all previously stored stories. + """ + for key in sorted(self.stories.keys()): + yield self.stories[key] diff --git a/tests/tasks/test_build.py b/tests/tasks/test_build.py new file mode 100644 index 0000000..817a6e6 --- /dev/null +++ b/tests/tasks/test_build.py @@ -0,0 +1,185 @@ +""" +Build task tests. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2020 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from pathlib import Path +from unittest.mock import call, patch, MagicMock + +import arrow +import pytest + +from fimfarchive.tasks import build, BuildTask +from fimfarchive.utils import AUTHOR_BLACKLIST + +from .conftest import DummyConverer, DummyFetcher + + +BLACKLISTED = sorted(AUTHOR_BLACKLIST)[0] + + +class TestBuildTask: + """ + Tests build task. + """ + + @pytest.fixture + def previous(self): + """ + Returns a `Fetcher` simulating Fimfarchive. + """ + fetcher = DummyFetcher() + + fetcher.add(key=1, date=1) + fetcher.add(key=2, date=2) + fetcher.add(key=3, date=3) + + fetcher.add(key=BLACKLISTED, date=BLACKLISTED) + + return fetcher + + @pytest.fixture + def upcoming(self): + """ + Returns a `Fetcher` simulating a directory. + """ + fetcher = DummyFetcher() + + fetcher.add(key=1, date=1, data=None) + fetcher.add(key=2, date=2, data=None) + fetcher.add(key=3, date=4) + fetcher.add(key=4, date=5) + + fetcher.add(key=BLACKLISTED, date=BLACKLISTED + 1) + + return fetcher + + @pytest.fixture + def result(self): + """ + Returns a `Fetcher` simulating the expected result. + """ + fetcher = DummyFetcher() + + fetcher.add(key=1, date=1) + fetcher.add(key=2, date=2) + fetcher.add(key=3, date=4) + fetcher.add(key=4, date=5) + + for story in fetcher: + story.meta['conversions'] += 1 + + return fetcher + + @pytest.fixture + def output(self, tmp_path): + """ + Returns the output path. + """ + output = tmp_path / 'output' + output.mkdir() + + return Path(output) + + @pytest.fixture + def extras_data(self): + """ + Returns the extra archive data. + """ + return [ + ('alpaca.txt', b"Alpacas are floofy!"), + ('pegasus.bin', b"\xF1u\xffy ponies!"), + ] + + @pytest.fixture + def extras_path(self, tmp_path, extras_data): + """ + Returns the extras path. + """ + extras = tmp_path / 'extras' + extras.mkdir() + + for name, data in extras_data: + path = extras / name + path.write_bytes(data) + + return Path(extras) + + @pytest.fixture + def task(self, output, upcoming, previous, extras_path): + """ + Returns a `BuildTask` instance. + """ + task = BuildTask( + output=output, + upcoming=upcoming, + previous=previous, + extras=extras_path, + ) + + with patch.object(task, 'convert', DummyConverer()): + yield task + + def test_path(self, task, output): + """ + Tests archive output path. + """ + date = arrow.utcnow().strftime("%Y%m%d") + name = f'fimfarchive-{date}.zip' + path = output / name + + assert path.resolve() == task.output.resolve() + + def test_extras(self, task, extras_data): + """ + Tests extra archive data. + """ + assert extras_data == sorted(task.extras) + + def test_generate(self, task, result): + """ + Tests content generator. + """ + for actual, expected in zip(task.generate(), result): + assert expected.meta == actual.meta + assert expected.data == actual.data + + def test_run(self, task): + """ + Tests writer calls. + """ + writer = MagicMock() + + manager = patch.object(build, 'FimfarchiveWriter', writer) + content = patch.object(task, 'generate', return_value=[1, 2, 4]) + + with manager, content: + task.run() + + assert writer.mock_calls == [ + call(task.output, task.extras), + call().__enter__(), + call().__enter__().write(1), + call().__enter__().write(2), + call().__enter__().write(4), + call().__exit__(None, None, None), + ]