mirror of
https://github.com/JockeTF/fimfarchive.git
synced 2024-11-22 13:28:00 +01:00
Use streaming text decoding while parsing index
This commit is contained in:
parent
d09cff4866
commit
77af31923e
1 changed files with 10 additions and 15 deletions
|
@ -22,6 +22,7 @@ Fetchers for Fimfarchive.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
|
||||||
|
import codecs
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
import gc
|
import gc
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
@ -32,6 +33,9 @@ from zipfile import ZipFile, BadZipFile
|
||||||
from fimfarchive.exceptions import InvalidStoryError, StorySourceError
|
from fimfarchive.exceptions import InvalidStoryError, StorySourceError
|
||||||
|
|
||||||
|
|
||||||
|
StreamReader = codecs.getreader('utf-8')
|
||||||
|
|
||||||
|
|
||||||
class Fetcher:
|
class Fetcher:
|
||||||
"""
|
"""
|
||||||
Abstract base class for story fetchers.
|
Abstract base class for story fetchers.
|
||||||
|
@ -217,26 +221,17 @@ class FimfarchiveFetcher(Fetcher):
|
||||||
raise StorySourceError("Archive is not a valid ZIP-file.") from e
|
raise StorySourceError("Archive is not a valid ZIP-file.") from e
|
||||||
|
|
||||||
try:
|
try:
|
||||||
byte_index = self.archive.read('index.json')
|
with self.archive.open('index.json') as fobj:
|
||||||
|
self.index = json.load(StreamReader(fobj))
|
||||||
except KeyError as e:
|
except KeyError as e:
|
||||||
raise StorySourceError("Archive is missing the index.") from e
|
raise StorySourceError("Archive is missing the index.") from e
|
||||||
|
except ValueError as e:
|
||||||
|
raise StorySourceError("Index is not valid JSON.") from e
|
||||||
|
except UnicodeDecodeError as e:
|
||||||
|
raise StorySourceError("Index is incorrectly encoded.") from e
|
||||||
except BadZipFile as e:
|
except BadZipFile as e:
|
||||||
raise StorySourceError("Archive is corrupt.") from e
|
raise StorySourceError("Archive is corrupt.") from e
|
||||||
|
|
||||||
try:
|
|
||||||
text_index = byte_index.decode()
|
|
||||||
except UnicodeDecodeError as e:
|
|
||||||
raise StorySourceError("Index is incorrectly encoded.") from e
|
|
||||||
|
|
||||||
del byte_index
|
|
||||||
gc.collect()
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.index = json.loads(text_index)
|
|
||||||
except ValueError as e:
|
|
||||||
raise StorySourceError("Index is not valid JSON.") from e
|
|
||||||
|
|
||||||
del text_index
|
|
||||||
gc.collect()
|
gc.collect()
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
|
|
Loading…
Reference in a new issue