From a19ea0319ee770124230765437e647c259625a48 Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Thu, 18 Oct 2018 19:10:25 +0200 Subject: [PATCH] Improve performance of Fimfarchive fetcher --- fimfarchive/fetchers/fimfarchive.py | 25 ++++++++++++++++--------- requirements.txt | 1 - setup.py | 1 - 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/fimfarchive/fetchers/fimfarchive.py b/fimfarchive/fetchers/fimfarchive.py index e0e3491..b1f8829 100644 --- a/fimfarchive/fetchers/fimfarchive.py +++ b/fimfarchive/fetchers/fimfarchive.py @@ -23,16 +23,14 @@ Fimfarchive fetcher. import json -from typing import cast, Any, Dict, IO, Iterator, Optional, Tuple, Union +from typing import Any, Dict, IO, Iterator, Optional, Tuple, Union from zipfile import ZipFile, BadZipFile -from boltons.cacheutils import LRU from jmespath import compile as jmes from fimfarchive.exceptions import InvalidStoryError, StorySourceError from fimfarchive.flavors import StorySource, DataFormat, MetaPurity from fimfarchive.stories import Story -from fimfarchive.utils import Empty from .base import Fetcher @@ -111,7 +109,7 @@ class FimfarchiveFetcher(Fetcher): except BadZipFile as e: raise StorySourceError("Archive is corrupt.") from e - self.paths = LRU() + self.paths = dict() self.is_open = True def load_index(self, source: IO[bytes]) -> Iterator[Tuple[int, str]]: @@ -196,13 +194,18 @@ class FimfarchiveFetcher(Fetcher): StorySourceError: If the fetcher is closed. """ key = self.validate(key) - path = self.paths.get(key, Empty) + path = self.paths.get(key) - if path is not Empty: - return cast(Optional[str], path) + if path is not None: + return path meta = self.fetch_meta(key) - return PATH.search(meta) + path = PATH.search(meta) + + if path is not None: + return path + + raise StorySourceError("Missing story path") def close(self) -> None: self.is_open = False @@ -232,7 +235,11 @@ class FimfarchiveFetcher(Fetcher): if key != actual: raise StorySourceError(f"Invalid ID for {key}: {actual}") - self.paths[key] = PATH.search(meta) + try: + archive = meta.get('archive', meta) + self.paths[key] = archive['path'] + except KeyError: + pass return meta diff --git a/requirements.txt b/requirements.txt index 3c59364..801a93e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ arrow bbcode blinker -boltons flake8 git+https://github.com/qvantel/jsonapi-client.git importlib_resources diff --git a/setup.py b/setup.py index 3d63e3d..dac08a9 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ setup( 'arrow', 'bbcode', 'blinker', - 'boltons', 'importlib_resources', 'jmespath', 'jsonapi-client',