PonyMirrors/fimfarchive
1
0
Fork 0
mirror of https://github.com/JockeTF/fimfarchive.git synced 2024-11-22 05:17:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Improve performance of Fimfarchive index loader

Browse source
This commit is contained in:
Joakim Soderlund 2018-11-22 15:26:29 +01:00
parent a19ea0319e
commit cdb968bbf0
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
fimfarchive/fetchers/fimfarchive.py
View file

@ -23,6 +23,7 @@ Fimfarchive fetcher.
import json import json
from io import BufferedReader
from typing import Any, Dict, IO, Iterator, Optional, Tuple, Union from typing import Any, Dict, IO, Iterator, Optional, Tuple, Union
from zipfile import ZipFile, BadZipFile from zipfile import ZipFile, BadZipFile
@ -47,6 +48,7 @@ __all__ = (
) )
BUFFER_SIZE = 8_000_000
PATH = jmes('archive.path || path') PATH = jmes('archive.path || path')
@ -103,7 +105,8 @@ class FimfarchiveFetcher(Fetcher):
try: try:
with self.archive.open('index.json') as fobj: with self.archive.open('index.json') as fobj:
self.index = dict(self.load_index(fobj)) reader = BufferedReader(fobj, BUFFER_SIZE) # type: ignore
self.index = dict(self.load_index(reader))
except KeyError as e: except KeyError as e:
raise StorySourceError("Archive is missing the index.") from e raise StorySourceError("Archive is missing the index.") from e
except BadZipFile as e: except BadZipFile as e:
@ -112,7 +115,7 @@ class FimfarchiveFetcher(Fetcher):
self.paths = dict() self.paths = dict()
self.is_open = True self.is_open = True
def load_index(self, source: IO[bytes]) -> Iterator[Tuple[int, str]]: def load_index(self, source: Iterator[bytes]) -> Iterator[Tuple[int, str]]:
""" """
Yields unparsed index items from a byte stream. Yields unparsed index items from a byte stream.