Add build task

fimfarchive/tasks/__init__.py

@@ -5,7 +5,7 @@ Tasks module.
 
 #
 # Fimfarchive, preserves stories from Fimfiction.
-# Copyright (C) 2015  Joakim Soderlund
+# Copyright (C) 2020  Joakim Soderlund
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,9 +22,11 @@ Tasks module.
 #
 
 
+from .build import BuildTask
 from .update import UpdateTask
 
 
 __all__ = (
+    'BuildTask',
     'UpdateTask',
 )

fimfarchive/tasks/build.py

@@ -0,0 +1,156 @@
+"""
+Build task.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2020  Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from pathlib import Path
+from typing import Iterable, Iterator, Optional, Tuple, Union
+
+import arrow
+
+from fimfarchive.converters import LocalUtcConverter
+from fimfarchive.exceptions import InvalidStoryError, StorySourceError
+from fimfarchive.fetchers import Fetcher
+from fimfarchive.stories import Story
+from fimfarchive.utils import is_blacklisted
+from fimfarchive.writers import FimfarchiveWriter
+
+
+__all__ = (
+    'BuildTask',
+)
+
+
+PathArg = Union[Path, str]
+
+
+class BuildTask:
+    """
+    Build a new version of the archive.
+    """
+
+    def __init__(
+            self,
+            output: PathArg,
+            upcoming: Iterable[Story],
+            previous: Fetcher = None,
+            extras: PathArg = None,
+            ) -> None:
+        """
+        Constructor.
+
+        Args:
+            output: Directory for the output
+            upcoming: Upcoming archive content
+            previous: Previous archive content
+            extras: Directory for extra files
+        """
+        self.previous = previous
+        self.upcoming = upcoming
+
+        self.convert = LocalUtcConverter()
+        self.output = self.get_output(output)
+        self.extras = self.get_extras(extras)
+
+    def get_output(self, directory: PathArg) -> Path:
+        """
+        Returns the complete path for the new archive.
+
+        Args:
+            directory: Path for the new archive
+        """
+        date = arrow.utcnow().format('YYYYMMDD')
+        path = Path(directory).resolve(True)
+        name = f'fimfarchive-{date}.zip'
+
+        return path / name
+
+    def get_extras(
+            self,
+            directory: Optional[PathArg]
+            ) -> Iterator[Tuple[str, bytes]]:
+        """
+        Yields extra archive data.
+
+        Args:
+            directory: Path to read extras from
+        """
+        if directory is None:
+            return ()
+
+        for path in Path(directory).iterdir():
+            yield path.name, path.read_bytes()
+
+    def revive(self, story: Story) -> Story:
+        """
+        Returns a story containing data from the previous archive.
+
+        Args:
+            story: The story to revive
+
+        Raises:
+            StorySourceError: If story data is missing
+        """
+        if self.previous is None:
+            raise StorySourceError("Missing previous fetcher.")
+
+        try:
+            revived = self.previous.fetch(story.key)
+        except InvalidStoryError as e:
+            raise StorySourceError("Missing revived story.") from e
+        else:
+            return story.merge(data=revived.data)
+
+    def resolve(self, story: Story) -> Story:
+        """
+        Returns a story guaranteed to contain data.
+
+        Args:
+            strory: The story to resolve
+
+        Raises:
+            StorySourceError: If story data is missing
+        """
+        try:
+            story.data
+        except InvalidStoryError:
+            return self.revive(story)
+        else:
+            return story
+
+    def generate(self) -> Iterator[Story]:
+        """
+        Yields stories for the new archive.
+        """
+        for story in self.upcoming:
+            if is_blacklisted(story):
+                continue
+
+            converted = self.convert(story)
+            resolved = self.resolve(converted)
+
+            yield resolved
+
+    def run(self):
+        with FimfarchiveWriter(self.output, self.extras) as writer:
+            for story in self.generate():
+                writer.write(story)

tests/tasks/conftest.py

@@ -22,11 +22,26 @@ Common task fixtures.
 #
 
 
+from copy import deepcopy
 from typing import Dict
 
 from fimfarchive.exceptions import InvalidStoryError
+from fimfarchive.converters import Converter
 from fimfarchive.fetchers import Fetcher
 from fimfarchive.stories import Story
+from fimfarchive.utils import Empty
+
+
+class DummyConverer(Converter):
+    """
+    Converter that increments a counter.
+    """
+
+    def __call__(self, story: Story) -> Story:
+        meta = deepcopy(story.meta)
+        meta['conversions'] += 1
+
+        return story.merge(meta=meta)
 
 
 class DummyFetcher(Fetcher):
@@ -40,28 +55,41 @@ class DummyFetcher(Fetcher):
         """
         self.stories: Dict[int, Story] = dict()
 
-    def add(self, key, date, flavors=()):
+    def add(self, key, date, flavors=(), data=Empty):
         """
         Adds a story to the fetcher.
         """
+        meta = {
+            'id': key,
+            'title': f't{key}',
+            'date_modified': date,
+            'conversions': 0,
+            'author': {
+                'id': key,
+                'name': f'n{key}'
+            },
+            'chapters': [
+                {'id': key},
+            ],
+        }
+
+        if data is Empty:
+            text = f'd{key}'
+            data = text.encode()
+
         story = Story(
             key=key,
+            fetcher=self,
+            meta=meta,
+            data=data,
             flavors=flavors,
-            data=f'Story {key}'.encode(),
-            meta={
-                'id': key,
-                'date_modified': date,
-                'chapters': [
-                    {'id': key},
-                ],
-            },
         )
 
         self.stories[key] = story
 
         return story
 
-    def fetch(self, key):
+    def fetch(self, key, prefetch_meta=None, prefetch_data=None):
         """
         Returns a previously stored story.
         """
@@ -69,3 +97,22 @@ class DummyFetcher(Fetcher):
             return self.stories[key]
         except KeyError:
             raise InvalidStoryError()
+
+    def fetch_data(self, key):
+        """
+        Raises exception for missing data.
+        """
+        raise InvalidStoryError()
+
+    def fetch_meta(self, key):
+        """
+        Raises exception for missing meta.
+        """
+        raise InvalidStoryError()
+
+    def __iter__(self):
+        """
+        Yields all previously stored stories.
+        """
+        for key in sorted(self.stories.keys()):
+            yield self.stories[key]

tests/tasks/test_build.py

@@ -0,0 +1,185 @@
+"""
+Build task tests.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2020  Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from pathlib import Path
+from unittest.mock import call, patch, MagicMock
+
+import arrow
+import pytest
+
+from fimfarchive.tasks import build, BuildTask
+from fimfarchive.utils import AUTHOR_BLACKLIST
+
+from .conftest import DummyConverer, DummyFetcher
+
+
+BLACKLISTED = sorted(AUTHOR_BLACKLIST)[0]
+
+
+class TestBuildTask:
+    """
+    Tests build task.
+    """
+
+    @pytest.fixture
+    def previous(self):
+        """
+        Returns a `Fetcher` simulating Fimfarchive.
+        """
+        fetcher = DummyFetcher()
+
+        fetcher.add(key=1, date=1)
+        fetcher.add(key=2, date=2)
+        fetcher.add(key=3, date=3)
+
+        fetcher.add(key=BLACKLISTED, date=BLACKLISTED)
+
+        return fetcher
+
+    @pytest.fixture
+    def upcoming(self):
+        """
+        Returns a `Fetcher` simulating a directory.
+        """
+        fetcher = DummyFetcher()
+
+        fetcher.add(key=1, date=1, data=None)
+        fetcher.add(key=2, date=2, data=None)
+        fetcher.add(key=3, date=4)
+        fetcher.add(key=4, date=5)
+
+        fetcher.add(key=BLACKLISTED, date=BLACKLISTED + 1)
+
+        return fetcher
+
+    @pytest.fixture
+    def result(self):
+        """
+        Returns a `Fetcher` simulating the expected result.
+        """
+        fetcher = DummyFetcher()
+
+        fetcher.add(key=1, date=1)
+        fetcher.add(key=2, date=2)
+        fetcher.add(key=3, date=4)
+        fetcher.add(key=4, date=5)
+
+        for story in fetcher:
+            story.meta['conversions'] += 1
+
+        return fetcher
+
+    @pytest.fixture
+    def output(self, tmp_path):
+        """
+        Returns the output path.
+        """
+        output = tmp_path / 'output'
+        output.mkdir()
+
+        return Path(output)
+
+    @pytest.fixture
+    def extras_data(self):
+        """
+        Returns the extra archive data.
+        """
+        return [
+            ('alpaca.txt', b"Alpacas are floofy!"),
+            ('pegasus.bin', b"\xF1u\xffy ponies!"),
+        ]
+
+    @pytest.fixture
+    def extras_path(self, tmp_path, extras_data):
+        """
+        Returns the extras path.
+        """
+        extras = tmp_path / 'extras'
+        extras.mkdir()
+
+        for name, data in extras_data:
+            path = extras / name
+            path.write_bytes(data)
+
+        return Path(extras)
+
+    @pytest.fixture
+    def task(self, output, upcoming, previous, extras_path):
+        """
+        Returns a `BuildTask` instance.
+        """
+        task = BuildTask(
+            output=output,
+            upcoming=upcoming,
+            previous=previous,
+            extras=extras_path,
+        )
+
+        with patch.object(task, 'convert', DummyConverer()):
+            yield task
+
+    def test_path(self, task, output):
+        """
+        Tests archive output path.
+        """
+        date = arrow.utcnow().strftime("%Y%m%d")
+        name = f'fimfarchive-{date}.zip'
+        path = output / name
+
+        assert path.resolve() == task.output.resolve()
+
+    def test_extras(self, task, extras_data):
+        """
+        Tests extra archive data.
+        """
+        assert extras_data == sorted(task.extras)
+
+    def test_generate(self, task, result):
+        """
+        Tests content generator.
+        """
+        for actual, expected in zip(task.generate(), result):
+            assert expected.meta == actual.meta
+            assert expected.data == actual.data
+
+    def test_run(self, task):
+        """
+        Tests writer calls.
+        """
+        writer = MagicMock()
+
+        manager = patch.object(build, 'FimfarchiveWriter', writer)
+        content = patch.object(task, 'generate', return_value=[1, 2, 4])
+
+        with manager, content:
+            task.run()
+
+        assert writer.mock_calls == [
+            call(task.output, task.extras),
+            call().__enter__(),
+            call().__enter__().write(1),
+            call().__enter__().write(2),
+            call().__enter__().write(4),
+            call().__exit__(None, None, None),
+        ]