PonyMirrors/fimfarchive
1
0
Fork 0
mirror of https://github.com/JockeTF/fimfarchive.git synced 2024-11-22 05:17:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Optionally compress index in Fimfarchive fetcher

Browse source
This commit is contained in:
Joakim Soderlund 2018-08-26 18:07:55 +02:00
parent df90cd5afa
commit 412deed8b5
1 changed files with 17 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
fimfarchive/fetchers/fimfarchive.py
View file

@ -37,6 +37,13 @@ from fimfarchive.utils import Empty
from .base import Fetcher
try:
from lz4.block import compress, decompress
except ModuleNotFoundError:
compress = lambda data: data # noqa
decompress = lambda data: data # noqa
__all__ = (
'FimfarchiveFetcher',
)
@ -124,20 +131,16 @@ class FimfarchiveFetcher(Fetcher):
if len(part) < 3:
continue
try:
line = part.decode().strip()
except UnicodeDecodeError as e:
raise StorySourceError("Incorrectly encoded index.") from e
line = part.strip()
key, meta = line.split(b':', 1)
key = key.strip(b' "')
meta = meta.strip(b' ,')
key, meta = line.split(':', 1)
key = key.strip(' "')
meta = meta.strip(' ,')
if meta[0] != '{' or meta[-1] != '}':
raise StorySourceError(f"Malformed index meta: {meta}")
if meta[0] != 123 or meta[-1] != 125:
raise StorySourceError(f"Malformed index meta: {key}")
try:
yield int(key), meta
yield int(key), compress(meta)
except ValueError as e:
raise StorySourceError(f"Malformed index key: {key}") from e
@ -218,7 +221,9 @@ class FimfarchiveFetcher(Fetcher):
raw = self.index[key]
try:
meta = json.loads(raw)
meta = json.loads(decompress(raw).decode())
except UnicodeDecodeError as e:
raise StorySourceError("Incorrectly encoded index.") from e
except ValueError as e:
raise StorySourceError(f"Malformed meta for {key}: {raw}") from e