PonyMirrors/fimfarchive
1
0
Fork 0
mirror of https://github.com/JockeTF/fimfarchive.git synced 2025-02-08 14:56:44 +01:00
Code Issues Projects Releases Packages Wiki Activity

Update fetcher to reflect changes in Fimfiction

Browse source
This commit is contained in:
Joakim Soderlund 2017-06-24 19:41:26 +02:00
parent 87a19fd2e9
commit 4bc306c6e0
2 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
fimfarchive/fetchers.py
View file

@ -147,8 +147,8 @@ class FimfictionFetcher(Fetcher):
prefetch_meta = True prefetch_meta = True
prefetch_data = False prefetch_data = False
data_path = 'https://www.fimfiction.net/download_story.php' data_path = 'https://www.fimfiction.net/story/download/{}/html'
meta_path = 'https://www.fimfiction.net/api/story.php' meta_path = 'https://www.fimfiction.net/api/story.php?story={}'
flavors = frozenset(( flavors = frozenset((
StorySource.FIMFICTION, StorySource.FIMFICTION,
@ -156,25 +156,28 @@ class FimfictionFetcher(Fetcher):
MetaPurity.DIRTY, MetaPurity.DIRTY,
)) ))
def get(self, url, **kwargs): def get(self, url):
""" """
Performs an HTTP GET request. Performs an HTTP GET request.
Args: Args:
url: Target of the HTTP request. url: Target of the HTTP request.
**kwargs: HTTP query parameters.
Returns: Returns:
Response: A new `Response` object. Response: A new `Response` object.
Raises: Raises:
StorySourceError: If the server does not return HTTP 200 OK. InvalidStoryError: If access to the resource was denied.
StorySourceError: If the request fails for any other reason.
""" """
try: try:
response = requests.get(url, params=kwargs, timeout=60) response = requests.get(url, timeout=60)
except OSError as e: except OSError as e:
raise StorySourceError("Could not read from server.") from e raise StorySourceError("Could not read from server.") from e
if response.status_code == 403:
raise InvalidStoryError("Access to resource was denied.")
if not response.ok: if not response.ok:
raise StorySourceError( raise StorySourceError(
"Server responded with HTTP {} {}." "Server responded with HTTP {} {}."
@ -184,13 +187,14 @@ class FimfictionFetcher(Fetcher):
return response return response
def fetch_data(self, key): def fetch_data(self, key):
response = self.get(self.data_path, story=key, html=True) url = self.data_path.format(key)
response = self.get(url)
data = response.content data = response.content
if len(data) == 0: if len(data) == 0:
raise InvalidStoryError("Server returned empty response body.") raise InvalidStoryError("Server returned empty response body.")
if b'<a name=\'1\'></a><h3>' not in data: if b'<h1><a name=\'1\'></a>' not in data:
raise InvalidStoryError("Server did not return any chapters.") raise InvalidStoryError("Server did not return any chapters.")
if not data.endswith(b'</html>'): if not data.endswith(b'</html>'):
@ -199,7 +203,8 @@ class FimfictionFetcher(Fetcher):
return data return data
def fetch_meta(self, key): def fetch_meta(self, key):
response = self.get(self.meta_path, story=key) url = self.meta_path.format(key)
response = self.get(url)
try: try:
meta = response.json() meta = response.json()

3
tests/test_fetchers.py
View file

@ -35,7 +35,7 @@ INVALID_STORY_KEY = 7
EMPTY_STORY_KEY = 8 EMPTY_STORY_KEY = 8
PROTECTED_STORY_KEY = 208799 PROTECTED_STORY_KEY = 208799
FIMFARCHIVE_PATH = 'fimfarchive-20161201.zip' FIMFARCHIVE_PATH = 'fimfarchive-20170601.zip'
class TestFetcher: class TestFetcher:
@ -205,6 +205,7 @@ class TestFimfictionFetcher:
data = fetcher.fetch_data(VALID_STORY_KEY) data = fetcher.fetch_data(VALID_STORY_KEY)
assert len(data) != 0 assert len(data) != 0
@pytest.mark.xfail(reason='knighty/fimfiction-issues#139')
def test_fetch_data_for_invalid_story(self, fetcher): def test_fetch_data_for_invalid_story(self, fetcher):
""" """
Tests `InvalidStoryError` is raised if story is invalid. Tests `InvalidStoryError` is raised if story is invalid.