PonyMirrors/fimfarchive
1
0
Fork 0
mirror of https://github.com/JockeTF/fimfarchive.git synced 2025-03-12 06:30:01 +01:00
Code Issues Projects Releases Packages Wiki Activity

TEMPORARY COMMIT - DO NOT MERGE

Browse source
This commit is contained in:
Joakim Soderlund 2025-02-22 17:49:52 +01:00
parent 6ef783aa98
commit f43d939333
1 changed files with 16 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
fimfarchive/fetchers/fimfiction2.py
View file

@ -26,6 +26,7 @@ import json
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections import OrderedDict, defaultdict from collections import OrderedDict, defaultdict
from copy import deepcopy from copy import deepcopy
from time import sleep
from typing import Any, Dict, Iterator, Optional, Set from typing import Any, Dict, Iterator, Optional, Set
from urllib.parse import urlencode from urllib.parse import urlencode
@ -36,6 +37,7 @@ from jsonapi_client.resourceobject import ResourceObject
from fimfarchive import __version__ as version from fimfarchive import __version__ as version
from fimfarchive.flavors import DataFormat, MetaFormat, MetaPurity, StorySource from fimfarchive.flavors import DataFormat, MetaFormat, MetaPurity, StorySource
from fimfarchive.utils import tqdm
from fimfarchive.exceptions import ( from fimfarchive.exceptions import (
FimfarchiveError, FimfarchiveError,
@ -302,9 +304,16 @@ class SingleRequester(Requester):
return response.resource return response.resource
def get_data(self, key: int) -> Iterator[ResourceObject]: def get_data(self, key: int) -> Iterator[ResourceObject]:
path = f'stories/{key}/chapters' list_path = f'stories/{key}/chapters'
response = self.get(key, path, DATA_PARAMS) list_params = deepcopy(DATA_PARAMS)
return response.resources list_params['fields[chapter]'].remove('content_html')
list_response = self.get(key, list_path, list_params)
for resource in tqdm(list_response.resources):
path = f"chapters/{resource.id}"
response = self.get(key, path, DATA_PARAMS)
yield response.resource
sleep(5)
class BulkRequester(Requester): class BulkRequester(Requester):
@ -685,7 +694,7 @@ class Fimfiction2Fetcher(Fetcher):
Fetcher for Fimfiction APIv2. Fetcher for Fimfiction APIv2.
""" """
prefetch_meta = True prefetch_meta = True
prefetch_data = False prefetch_data = True
flavors = frozenset(( flavors = frozenset((
StorySource.FIMFICTION, StorySource.FIMFICTION,
@ -703,12 +712,14 @@ class Fimfiction2Fetcher(Fetcher):
bulk_meta: Toggles bulk fetching of story meta. bulk_meta: Toggles bulk fetching of story meta.
bulk_data: Toggles bulk fetching of story data. bulk_data: Toggles bulk fetching of story data.
""" """
assert not bulk_meta
assert not bulk_data
client = ApiClient(token) client = ApiClient(token)
self.extract_meta = MetaDocumentifier() self.extract_meta = MetaDocumentifier()
self.extract_chapter = Documentifier() self.extract_chapter = Documentifier()
self.verify_meta = BetaFormatVerifier.from_meta_params() self.verify_meta = BetaFormatVerifier.from_meta_params()
self.verify_chapter = BetaFormatVerifier.from_data_params() self.verify_chapter = BetaFormatVerifier.from_data_params()
self.requester = RoutedRequester(client, bulk_meta, bulk_data) self.requester = SingleRequester(client)
def fetch_meta(self, key: int) -> Dict[str, Any]: def fetch_meta(self, key: int) -> Dict[str, Any]:
resource = self.requester.get_meta(int(key)) resource = self.requester.get_meta(int(key))