From 77af31923e344268fbba41cff5884156fe1412b6 Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Sun, 18 Dec 2016 00:06:01 +0100 Subject: [PATCH] Use streaming text decoding while parsing index --- fimfarchive/fetchers.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/fimfarchive/fetchers.py b/fimfarchive/fetchers.py index e14f279..5cf1d8b 100644 --- a/fimfarchive/fetchers.py +++ b/fimfarchive/fetchers.py @@ -22,6 +22,7 @@ Fetchers for Fimfarchive. # +import codecs from copy import deepcopy import gc from io import BytesIO @@ -32,6 +33,9 @@ from zipfile import ZipFile, BadZipFile from fimfarchive.exceptions import InvalidStoryError, StorySourceError +StreamReader = codecs.getreader('utf-8') + + class Fetcher: """ Abstract base class for story fetchers. @@ -217,26 +221,17 @@ class FimfarchiveFetcher(Fetcher): raise StorySourceError("Archive is not a valid ZIP-file.") from e try: - byte_index = self.archive.read('index.json') + with self.archive.open('index.json') as fobj: + self.index = json.load(StreamReader(fobj)) except KeyError as e: raise StorySourceError("Archive is missing the index.") from e + except ValueError as e: + raise StorySourceError("Index is not valid JSON.") from e + except UnicodeDecodeError as e: + raise StorySourceError("Index is incorrectly encoded.") from e except BadZipFile as e: raise StorySourceError("Archive is corrupt.") from e - try: - text_index = byte_index.decode() - except UnicodeDecodeError as e: - raise StorySourceError("Index is incorrectly encoded.") from e - - del byte_index - gc.collect() - - try: - self.index = json.loads(text_index) - except ValueError as e: - raise StorySourceError("Index is not valid JSON.") from e - - del text_index gc.collect() def close(self):