diff --git a/fimfarchive/commands/__init__.py b/fimfarchive/commands/__init__.py index 47e813d..1e2b65f 100644 --- a/fimfarchive/commands/__init__.py +++ b/fimfarchive/commands/__init__.py @@ -25,6 +25,7 @@ Command module. from .base import Command from .build import BuildCommand from .root import RootCommand +from .render import RenderCommand from .update import UpdateCommand @@ -32,5 +33,6 @@ __all__ = ( 'Command', 'RootCommand', 'BuildCommand', + 'RenderCommand', 'UpdateCommand', ) diff --git a/fimfarchive/commands/render.py b/fimfarchive/commands/render.py new file mode 100644 index 0000000..e3fbd5e --- /dev/null +++ b/fimfarchive/commands/render.py @@ -0,0 +1,113 @@ +""" +Render command. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2019 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from argparse import ArgumentParser, Namespace +from pathlib import Path + +import arrow + +from fimfarchive.signals import SignalReceiver +from fimfarchive.tasks import RenderTask +from fimfarchive.utils import tqdm + +from .base import Command + + +__all__ = ( + 'RenderCommand', +) + + +class RenderPrinter(SignalReceiver): + + def on_enter(self, sender, keys, workers, spec): + self.entered = arrow.utcnow() + print(f"\nStarted: {self.entered}") + print(f"Directory: {spec.worktree}") + print(f"Stories: {len(keys)}") + print(f"Workers: {workers}\n") + self.tqdm = tqdm(total=len(keys)) + + def on_success(self, sender, key): + self.tqdm.update() + + def on_failure(self, sender, key, error): + self.tqdm.write(f"[{key:6}] {error}") + self.tqdm.update() + + def on_exit(self, sender, converted, remaining): + self.tqdm.close() + self.exited = arrow.utcnow() + print(f"\nDone: {self.exited}") + print(f"Duration: {self.exited - self.entered}") + print(f"Converted: {len(converted)}") + print(f"Remaining: {len(remaining)}\n") + + +class RenderCommand(Command): + """ + Renders updates as EPUB files. + """ + + @property + def parser(self) -> ArgumentParser: + """ + Returns a command line arguments parser. + """ + parser = ArgumentParser( + prog='', + description=self.__doc__, + ) + + parser.add_argument( + '--worktree', + help="Working directory for the archive", + metavar='PATH', + default='worktree', + ) + + return parser + + def configure(self, opts: Namespace) -> RenderTask: + """ + Returns a configured task instance. + + Args: + opts: Parsed command line arguments. + """ + worktree = Path(opts.worktree).resolve() + + if not worktree.is_dir(): + self.parser.error(f"No such directory: {worktree}") + + return RenderTask(str(worktree)) + + def __call__(self, *args): + opts = self.parser.parse_args(args) + task = self.configure(opts) + + with RenderPrinter(task): + task.run() + + return 0 diff --git a/fimfarchive/commands/root.py b/fimfarchive/commands/root.py index cf92524..787299b 100644 --- a/fimfarchive/commands/root.py +++ b/fimfarchive/commands/root.py @@ -26,6 +26,7 @@ from typing import Dict, Type from .base import Command from .build import BuildCommand +from .render import RenderCommand from .update import UpdateCommand @@ -40,6 +41,7 @@ class RootCommand(Command): """ commands: Dict[str, Type[Command]] = { 'build': BuildCommand, + 'render': RenderCommand, 'update': UpdateCommand, } diff --git a/fimfarchive/tasks/__init__.py b/fimfarchive/tasks/__init__.py index b2bb66a..db3e70b 100644 --- a/fimfarchive/tasks/__init__.py +++ b/fimfarchive/tasks/__init__.py @@ -23,10 +23,12 @@ Tasks module. from .build import BuildTask +from .render import RenderTask from .update import UpdateTask __all__ = ( 'BuildTask', + 'RenderTask', 'UpdateTask', ) diff --git a/fimfarchive/tasks/render.py b/fimfarchive/tasks/render.py new file mode 100644 index 0000000..bb6ee3e --- /dev/null +++ b/fimfarchive/tasks/render.py @@ -0,0 +1,171 @@ +""" +Render task. +""" + + +# +# Fimfarchive, preserves stories from Fimfiction. +# Copyright (C) 2019 Joakim Soderlund +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from multiprocessing import Pool +from os import cpu_count +from pathlib import Path +from typing import List, Optional, Tuple + +from fimfarchive.converters import JsonFpubConverter, FpubEpubConverter +from fimfarchive.fetchers import DirectoryFetcher +from fimfarchive.flavors import DataFormat, MetaFormat +from fimfarchive.mappers import MetaFormatMapper +from fimfarchive.signals import Signal, SignalSender +from fimfarchive.stories import Story +from fimfarchive.writers import DirectoryWriter + + +__all__ = ( + 'RenderTask', +) + + +WORKERS = 4 +WORKTREE = 'worktree' + + +class PathSpec: + def __init__(self, worktree: str) -> None: + self.worktree = Path(worktree) + self.source = self.worktree / 'update' + self.target = self.worktree / 'render' + self.meta = self.source / 'meta' + self.json = self.source / 'json' + self.epub = self.target / 'epub' + self.logs = self.target / 'logs' + + def verify_dir(self, path: Path) -> None: + if not path.is_dir(): + raise ValueError(f"Missing dir: {path}") + + def create_dir(self, path: Path) -> None: + path.mkdir(mode=0o755, parents=True, exist_ok=True) + + def prepare(self) -> None: + self.verify_dir(self.meta) + self.verify_dir(self.json) + self.create_dir(self.epub) + self.create_dir(self.logs) + + +class Executor: + initialized: bool = False + + def __init__(self, worktree: str) -> None: + self.worktree = worktree + + def initialize(self) -> None: + path = PathSpec(self.worktree) + + self.fetcher = DirectoryFetcher( + meta_path=path.meta, + data_path=path.json, + flavors=[DataFormat.JSON], + ) + + self.writer = DirectoryWriter( + data_path=str(path.epub), + make_dirs=False, + ) + + self.to_fpub = JsonFpubConverter() + self.to_epub = FpubEpubConverter(str(path.logs)) + self.get_meta_format = MetaFormatMapper() + self.initialized = True + + def fetch(self, key: int) -> Story: + story = self.fetcher.fetch(key) + + if MetaFormat.BETA in story.flavors: + raise ValueError("Flavor should not be static: {MetaFormat.BETA}") + + if self.get_meta_format(story) != MetaFormat.BETA: + raise ValueError("Flavor could not be detected: {MetaFormat.BETA}") + + story.flavors.add(MetaFormat.BETA) + + return story + + def apply(self, key: int) -> None: + json = self.fetch(key) + fpub = self.to_fpub(json) + epub = self.to_epub(fpub) + self.writer.write(epub) + + def __call__(self, key: int) -> Tuple[int, Optional[str]]: + if not self.initialized: + self.initialize() + + try: + self.apply(key) + except Exception as e: + return key, f"{type(e).__name__}: {e}" + else: + return key, None + + +class RenderTask(SignalSender): + on_enter = Signal('keys', 'workers', 'spec') + on_exit = Signal('converted', 'remaining') + on_success = Signal('key') + on_failure = Signal('key', 'error') + + def __init__(self, worktree: str = WORKTREE) -> None: + """ + Constructor. + """ + super().__init__() + self.worktree = worktree + + def subtasks(self, spec: PathSpec) -> List[int]: + sources = {int(path.name) for path in spec.json.iterdir()} + targets = {int(path.name) for path in spec.epub.iterdir()} + + return sorted(sources - targets) + + def run(self) -> None: + spec = PathSpec(self.worktree) + + spec.prepare() + keys = self.subtasks(spec) + func = Executor(self.worktree) + + workers = cpu_count() or WORKERS + converted: List[int] = list() + remaining: List[int] = list() + + self.on_enter(keys, workers, spec) + + with Pool(workers) as pool: + mapper = pool.imap_unordered(func, keys) + + for key, error in mapper: + if error is None: + converted.append(key) + self.on_success(key) + else: + remaining.append(key) + self.on_failure(key, error) + + self.on_exit(converted, remaining)