diff --git a/fimfarchive/tasks/__init__.py b/fimfarchive/tasks/__init__.py
index 61e4348..b2bb66a 100644
--- a/fimfarchive/tasks/__init__.py
+++ b/fimfarchive/tasks/__init__.py
@@ -5,7 +5,7 @@ Tasks module.
#
# Fimfarchive, preserves stories from Fimfiction.
-# Copyright (C) 2015 Joakim Soderlund
+# Copyright (C) 2020 Joakim Soderlund
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -22,9 +22,11 @@ Tasks module.
#
+from .build import BuildTask
from .update import UpdateTask
__all__ = (
+ 'BuildTask',
'UpdateTask',
)
diff --git a/fimfarchive/tasks/build.py b/fimfarchive/tasks/build.py
new file mode 100644
index 0000000..5d09883
--- /dev/null
+++ b/fimfarchive/tasks/build.py
@@ -0,0 +1,156 @@
+"""
+Build task.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2020 Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+
+from pathlib import Path
+from typing import Iterable, Iterator, Optional, Tuple, Union
+
+import arrow
+
+from fimfarchive.converters import LocalUtcConverter
+from fimfarchive.exceptions import InvalidStoryError, StorySourceError
+from fimfarchive.fetchers import Fetcher
+from fimfarchive.stories import Story
+from fimfarchive.utils import is_blacklisted
+from fimfarchive.writers import FimfarchiveWriter
+
+
+__all__ = (
+ 'BuildTask',
+)
+
+
+PathArg = Union[Path, str]
+
+
+class BuildTask:
+ """
+ Build a new version of the archive.
+ """
+
+ def __init__(
+ self,
+ output: PathArg,
+ upcoming: Iterable[Story],
+ previous: Fetcher = None,
+ extras: PathArg = None,
+ ) -> None:
+ """
+ Constructor.
+
+ Args:
+ output: Directory for the output
+ upcoming: Upcoming archive content
+ previous: Previous archive content
+ extras: Directory for extra files
+ """
+ self.previous = previous
+ self.upcoming = upcoming
+
+ self.convert = LocalUtcConverter()
+ self.output = self.get_output(output)
+ self.extras = self.get_extras(extras)
+
+ def get_output(self, directory: PathArg) -> Path:
+ """
+ Returns the complete path for the new archive.
+
+ Args:
+ directory: Path for the new archive
+ """
+ date = arrow.utcnow().format('YYYYMMDD')
+ path = Path(directory).resolve(True)
+ name = f'fimfarchive-{date}.zip'
+
+ return path / name
+
+ def get_extras(
+ self,
+ directory: Optional[PathArg]
+ ) -> Iterator[Tuple[str, bytes]]:
+ """
+ Yields extra archive data.
+
+ Args:
+ directory: Path to read extras from
+ """
+ if directory is None:
+ return ()
+
+ for path in Path(directory).iterdir():
+ yield path.name, path.read_bytes()
+
+ def revive(self, story: Story) -> Story:
+ """
+ Returns a story containing data from the previous archive.
+
+ Args:
+ story: The story to revive
+
+ Raises:
+ StorySourceError: If story data is missing
+ """
+ if self.previous is None:
+ raise StorySourceError("Missing previous fetcher.")
+
+ try:
+ revived = self.previous.fetch(story.key)
+ except InvalidStoryError as e:
+ raise StorySourceError("Missing revived story.") from e
+ else:
+ return story.merge(data=revived.data)
+
+ def resolve(self, story: Story) -> Story:
+ """
+ Returns a story guaranteed to contain data.
+
+ Args:
+ strory: The story to resolve
+
+ Raises:
+ StorySourceError: If story data is missing
+ """
+ try:
+ story.data
+ except InvalidStoryError:
+ return self.revive(story)
+ else:
+ return story
+
+ def generate(self) -> Iterator[Story]:
+ """
+ Yields stories for the new archive.
+ """
+ for story in self.upcoming:
+ if is_blacklisted(story):
+ continue
+
+ converted = self.convert(story)
+ resolved = self.resolve(converted)
+
+ yield resolved
+
+ def run(self):
+ with FimfarchiveWriter(self.output, self.extras) as writer:
+ for story in self.generate():
+ writer.write(story)
diff --git a/tests/tasks/conftest.py b/tests/tasks/conftest.py
index 918e9ed..04afc92 100644
--- a/tests/tasks/conftest.py
+++ b/tests/tasks/conftest.py
@@ -22,11 +22,26 @@ Common task fixtures.
#
+from copy import deepcopy
from typing import Dict
from fimfarchive.exceptions import InvalidStoryError
+from fimfarchive.converters import Converter
from fimfarchive.fetchers import Fetcher
from fimfarchive.stories import Story
+from fimfarchive.utils import Empty
+
+
+class DummyConverer(Converter):
+ """
+ Converter that increments a counter.
+ """
+
+ def __call__(self, story: Story) -> Story:
+ meta = deepcopy(story.meta)
+ meta['conversions'] += 1
+
+ return story.merge(meta=meta)
class DummyFetcher(Fetcher):
@@ -40,28 +55,41 @@ class DummyFetcher(Fetcher):
"""
self.stories: Dict[int, Story] = dict()
- def add(self, key, date, flavors=()):
+ def add(self, key, date, flavors=(), data=Empty):
"""
Adds a story to the fetcher.
"""
+ meta = {
+ 'id': key,
+ 'title': f't{key}',
+ 'date_modified': date,
+ 'conversions': 0,
+ 'author': {
+ 'id': key,
+ 'name': f'n{key}'
+ },
+ 'chapters': [
+ {'id': key},
+ ],
+ }
+
+ if data is Empty:
+ text = f'd{key}'
+ data = text.encode()
+
story = Story(
key=key,
+ fetcher=self,
+ meta=meta,
+ data=data,
flavors=flavors,
- data=f'Story {key}'.encode(),
- meta={
- 'id': key,
- 'date_modified': date,
- 'chapters': [
- {'id': key},
- ],
- },
)
self.stories[key] = story
return story
- def fetch(self, key):
+ def fetch(self, key, prefetch_meta=None, prefetch_data=None):
"""
Returns a previously stored story.
"""
@@ -69,3 +97,22 @@ class DummyFetcher(Fetcher):
return self.stories[key]
except KeyError:
raise InvalidStoryError()
+
+ def fetch_data(self, key):
+ """
+ Raises exception for missing data.
+ """
+ raise InvalidStoryError()
+
+ def fetch_meta(self, key):
+ """
+ Raises exception for missing meta.
+ """
+ raise InvalidStoryError()
+
+ def __iter__(self):
+ """
+ Yields all previously stored stories.
+ """
+ for key in sorted(self.stories.keys()):
+ yield self.stories[key]
diff --git a/tests/tasks/test_build.py b/tests/tasks/test_build.py
new file mode 100644
index 0000000..817a6e6
--- /dev/null
+++ b/tests/tasks/test_build.py
@@ -0,0 +1,185 @@
+"""
+Build task tests.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2020 Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+
+from pathlib import Path
+from unittest.mock import call, patch, MagicMock
+
+import arrow
+import pytest
+
+from fimfarchive.tasks import build, BuildTask
+from fimfarchive.utils import AUTHOR_BLACKLIST
+
+from .conftest import DummyConverer, DummyFetcher
+
+
+BLACKLISTED = sorted(AUTHOR_BLACKLIST)[0]
+
+
+class TestBuildTask:
+ """
+ Tests build task.
+ """
+
+ @pytest.fixture
+ def previous(self):
+ """
+ Returns a `Fetcher` simulating Fimfarchive.
+ """
+ fetcher = DummyFetcher()
+
+ fetcher.add(key=1, date=1)
+ fetcher.add(key=2, date=2)
+ fetcher.add(key=3, date=3)
+
+ fetcher.add(key=BLACKLISTED, date=BLACKLISTED)
+
+ return fetcher
+
+ @pytest.fixture
+ def upcoming(self):
+ """
+ Returns a `Fetcher` simulating a directory.
+ """
+ fetcher = DummyFetcher()
+
+ fetcher.add(key=1, date=1, data=None)
+ fetcher.add(key=2, date=2, data=None)
+ fetcher.add(key=3, date=4)
+ fetcher.add(key=4, date=5)
+
+ fetcher.add(key=BLACKLISTED, date=BLACKLISTED + 1)
+
+ return fetcher
+
+ @pytest.fixture
+ def result(self):
+ """
+ Returns a `Fetcher` simulating the expected result.
+ """
+ fetcher = DummyFetcher()
+
+ fetcher.add(key=1, date=1)
+ fetcher.add(key=2, date=2)
+ fetcher.add(key=3, date=4)
+ fetcher.add(key=4, date=5)
+
+ for story in fetcher:
+ story.meta['conversions'] += 1
+
+ return fetcher
+
+ @pytest.fixture
+ def output(self, tmp_path):
+ """
+ Returns the output path.
+ """
+ output = tmp_path / 'output'
+ output.mkdir()
+
+ return Path(output)
+
+ @pytest.fixture
+ def extras_data(self):
+ """
+ Returns the extra archive data.
+ """
+ return [
+ ('alpaca.txt', b"Alpacas are floofy!"),
+ ('pegasus.bin', b"\xF1u\xffy ponies!"),
+ ]
+
+ @pytest.fixture
+ def extras_path(self, tmp_path, extras_data):
+ """
+ Returns the extras path.
+ """
+ extras = tmp_path / 'extras'
+ extras.mkdir()
+
+ for name, data in extras_data:
+ path = extras / name
+ path.write_bytes(data)
+
+ return Path(extras)
+
+ @pytest.fixture
+ def task(self, output, upcoming, previous, extras_path):
+ """
+ Returns a `BuildTask` instance.
+ """
+ task = BuildTask(
+ output=output,
+ upcoming=upcoming,
+ previous=previous,
+ extras=extras_path,
+ )
+
+ with patch.object(task, 'convert', DummyConverer()):
+ yield task
+
+ def test_path(self, task, output):
+ """
+ Tests archive output path.
+ """
+ date = arrow.utcnow().strftime("%Y%m%d")
+ name = f'fimfarchive-{date}.zip'
+ path = output / name
+
+ assert path.resolve() == task.output.resolve()
+
+ def test_extras(self, task, extras_data):
+ """
+ Tests extra archive data.
+ """
+ assert extras_data == sorted(task.extras)
+
+ def test_generate(self, task, result):
+ """
+ Tests content generator.
+ """
+ for actual, expected in zip(task.generate(), result):
+ assert expected.meta == actual.meta
+ assert expected.data == actual.data
+
+ def test_run(self, task):
+ """
+ Tests writer calls.
+ """
+ writer = MagicMock()
+
+ manager = patch.object(build, 'FimfarchiveWriter', writer)
+ content = patch.object(task, 'generate', return_value=[1, 2, 4])
+
+ with manager, content:
+ task.run()
+
+ assert writer.mock_calls == [
+ call(task.output, task.extras),
+ call().__enter__(),
+ call().__enter__().write(1),
+ call().__enter__().write(2),
+ call().__enter__().write(4),
+ call().__exit__(None, None, None),
+ ]