Use streaming text decoding while parsing index

This commit is contained in:
Joakim Soderlund 2016-12-18 00:06:01 +01:00
parent d09cff4866
commit 77af31923e

View file

@ -22,6 +22,7 @@ Fetchers for Fimfarchive.
# #
import codecs
from copy import deepcopy from copy import deepcopy
import gc import gc
from io import BytesIO from io import BytesIO
@ -32,6 +33,9 @@ from zipfile import ZipFile, BadZipFile
from fimfarchive.exceptions import InvalidStoryError, StorySourceError from fimfarchive.exceptions import InvalidStoryError, StorySourceError
StreamReader = codecs.getreader('utf-8')
class Fetcher: class Fetcher:
""" """
Abstract base class for story fetchers. Abstract base class for story fetchers.
@ -217,26 +221,17 @@ class FimfarchiveFetcher(Fetcher):
raise StorySourceError("Archive is not a valid ZIP-file.") from e raise StorySourceError("Archive is not a valid ZIP-file.") from e
try: try:
byte_index = self.archive.read('index.json') with self.archive.open('index.json') as fobj:
self.index = json.load(StreamReader(fobj))
except KeyError as e: except KeyError as e:
raise StorySourceError("Archive is missing the index.") from e raise StorySourceError("Archive is missing the index.") from e
except ValueError as e:
raise StorySourceError("Index is not valid JSON.") from e
except UnicodeDecodeError as e:
raise StorySourceError("Index is incorrectly encoded.") from e
except BadZipFile as e: except BadZipFile as e:
raise StorySourceError("Archive is corrupt.") from e raise StorySourceError("Archive is corrupt.") from e
try:
text_index = byte_index.decode()
except UnicodeDecodeError as e:
raise StorySourceError("Index is incorrectly encoded.") from e
del byte_index
gc.collect()
try:
self.index = json.loads(text_index)
except ValueError as e:
raise StorySourceError("Index is not valid JSON.") from e
del text_index
gc.collect() gc.collect()
def close(self): def close(self):