From e80a3d1c2c0da1ea7c04513ae3bd399b479c439f Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Sat, 13 Oct 2018 19:59:50 +0200 Subject: [PATCH] Add mapper for story slug --- fimfarchive/mappers.py | 154 ++++++++++++++++++++++++++++++++++++++++- tests/test_mappers.py | 127 ++++++++++++++++++++++++++++++++- 2 files changed, 275 insertions(+), 6 deletions(-) diff --git a/fimfarchive/mappers.py b/fimfarchive/mappers.py index 24ac378..bd4e0cf 100644 --- a/fimfarchive/mappers.py +++ b/fimfarchive/mappers.py @@ -5,7 +5,7 @@ Mappers for Fimfarchive. # # Fimfarchive, preserves stories from Fimfiction. -# Copyright (C) 2015 Joakim Soderlund +# Copyright (C) 2018 Joakim Soderlund # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,13 +23,15 @@ Mappers for Fimfarchive. import os +import string from abc import abstractmethod -from typing import Dict, Generic, Optional, Set, TypeVar +from html import unescape +from typing import Dict, Generic, Optional, Set, TypeVar, Union from arrow import api as arrow, Arrow from fimfarchive.exceptions import InvalidStoryError -from fimfarchive.flavors import MetaFormat +from fimfarchive.flavors import DataFormat, MetaFormat from fimfarchive.stories import Story from fimfarchive.utils import find_flavor @@ -39,6 +41,7 @@ __all__ = ( 'StaticMapper', 'StoryDateMapper', 'StoryPathMapper', + 'StorySlugMapper', ) @@ -119,6 +122,151 @@ class StoryPathMapper(Mapper[str]): return os.path.join(directory, key) +class StorySlugMapper(Mapper[str]): + """ + Returns a slug-based file path for a story. + """ + + def __init__(self, template: str = None) -> None: + """ + Constructor. + + Args: + template: Format string for the path. + """ + self.template = '{extension}/{group}/{author}/{story}.{extension}' + self.whitelist = set(string.ascii_letters + string.digits) + self.groupings = set(string.ascii_lowercase) + self.ignore = {'\''} + self.spacing = '_' + self.part_limit = 112 + self.slug_limit = 255 + + if template is not None: + self.template = template + + def slugify(self, text: str) -> str: + """ + Creates a slug from any text. + + Args: + text: The text to slufigy. + + Returns: + A slugified version of the text. + """ + chars = [self.spacing] + + for char in unescape(str(text)): + if char in self.whitelist: + chars.append(char) + elif char not in self.ignore and chars[-1] != self.spacing: + chars.append(self.spacing) + + slug = ''.join(chars).strip(self.spacing) + + if self.part_limit < len(slug): + limit = self.part_limit + 1 + chars = slug[:limit].split(self.spacing) + slug = self.spacing.join(chars[:-1]) + + return slug.lower() + + def join(self, key: int, slug: Optional[str]) -> str: + """ + Appends a key to a slug. + + Args: + key: The key to append. + slug: The target slug. + + Returns: + A string with both slug and key. + """ + key = int(key) + + if key < 0: + raise ValueError("Key must not be negative.") + + if slug: + return f'{slug}-{key}' + else: + return f'{key}' + + def group(self, slug: str) -> str: + """ + Returns the group for part of a path. + + Args: + slug: The slug to group. + + Return: + A group for the slug. + """ + path = slug[:1] + + if path not in self.groupings: + path = self.spacing + + return path + + def classify(self, story: Story) -> str: + """ + Returns a file extension for the story. + + Args: + story: The story to classify. + + Returns: + A file extension. + """ + for flavor in story.flavors: + if isinstance(flavor, DataFormat): + return flavor.name.lower() + + return 'data' + + def generate(self, story: Story) -> Dict[str, Union[int, str]]: + """ + Returns the parts of a path. + + Args: + story: The story to generate parts from. + + Returns: + The parts of the path. + """ + story_meta = story.meta + author_meta = story.meta['author'] + + story_slug = self.slugify(story_meta['title']) + author_slug = self.slugify(author_meta['name']) + story_path = self.join(story_meta['id'], story_slug) + author_path = self.join(author_meta['id'], author_slug) + group_path = self.group(author_path) + extension = self.classify(story) + + return { + 'group': group_path, + 'author': author_path, + 'author_key': author_meta['id'], + 'author_slug': author_path, + 'story': story_path, + 'story_key': story_meta['id'], + 'story_slug': story_slug, + 'extension': extension, + } + + def __call__(self, story: Story) -> str: + parts = self.generate(story) + path = self.template.format(**parts) + + if self.slug_limit < len(path): + raise ValueError("Path too long: {}".format(path)) + + return path + + class MetaFormatMapper(Mapper[Optional[MetaFormat]]): """ Guesses the meta format of stories. diff --git a/tests/test_mappers.py b/tests/test_mappers.py index ac2a550..880ceb0 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -5,7 +5,7 @@ Mapper tests. # # Fimfarchive, preserves stories from Fimfiction. -# Copyright (C) 2015 Joakim Soderlund +# Copyright (C) 2018 Joakim Soderlund # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,9 +29,10 @@ from unittest.mock import patch, MagicMock, PropertyMock import pytest from fimfarchive.exceptions import InvalidStoryError -from fimfarchive.flavors import MetaFormat +from fimfarchive.flavors import DataFormat, MetaFormat from fimfarchive.mappers import ( - MetaFormatMapper, StaticMapper, StoryDateMapper, StoryPathMapper + MetaFormatMapper, StaticMapper, StoryDateMapper, + StoryPathMapper, StorySlugMapper ) from fimfarchive.stories import Story @@ -281,6 +282,126 @@ class TestStoryPathMapper: assert story.key.__str__.called_once_with() +class TestStorySlugMapper: + """ + StorySlugMapper tests. + """ + + @pytest.fixture + def mapper(self) -> StorySlugMapper: + """ + Returns a mapper instance. + """ + return StorySlugMapper() + + @pytest.fixture + def story(self, story: Story) -> Story: + """ + Returns a story instance. + """ + meta = { + 'id': 1337, + 'title': 'title', + 'author': { + 'id': 42, + 'name': 'name', + }, + } + + return story.merge( + meta=meta, + flavors=[DataFormat.EPUB] + ) + + def test_mapping(self, mapper, story): + """ + Tests a simple slug mapping. + """ + assert mapper(story) == 'epub/n/name-42/title-1337.epub' + + def test_custom_mapping(self, story): + """ + Tests a slug mapping with a custom template. + """ + mapper = StorySlugMapper('{story}.{extension}') + + assert mapper(story) == 'title-1337.epub' + + @pytest.mark.parametrize('text,result', ( + ('Project Sunflower: Harmony', 'project_sunflower_harmony'), + ('The Enchanted Library', 'the_enchanted_library'), + ('Hurricane\'s Way', 'hurricanes_way'), + ('Sharers\' Day', 'sharers_day'), + ('Paca ' * 32, ('paca_' * 22)[:-1]), + ('Paca' * 32, ''), + (None, 'none'), + (' ', ''), + (' ', ''), + ('', ''), + )) + def test_slugify(self, mapper, text, result): + """ + Tests creating a slug for a title. + """ + assert mapper.slugify(text) == result + + @pytest.mark.parametrize('key,slug,result', ( + (16, 'slug', 'slug-16'), + (32, None, '32'), + (64, '', '64'), + )) + def test_join(self, mapper, key, slug, result): + """ + Tests joining a slug with a key. + """ + assert mapper.join(key, slug) == result + + def test_join_with_negative_key(self, mapper): + """ + Tests `ValueError` is raised when joining a negative key. + """ + with pytest.raises(ValueError): + mapper.join('-1', 'slug') + + @pytest.mark.parametrize('slug,result', ( + ('alpaca', 'a'), + ('pony', 'p'), + ('42', '_'), + (' ', '_'), + (' ', '_'), + ('', '_'), + )) + def test_group(self, mapper, slug, result): + """ + Tests grouping of a slug. + """ + assert mapper.group(slug) == result + + @pytest.mark.parametrize('flavors,result', ( + ({MetaFormat.BETA, DataFormat.EPUB}, 'epub'), + ({DataFormat.EPUB}, 'epub'), + ({DataFormat.HTML}, 'html'), + ({MetaFormat.BETA}, 'data'), + ({}, 'data'), + )) + def test_classify(self, mapper, story, flavors, result): + """ + Tests classify with a story. + """ + story = story.merge(flavors=flavors) + + assert mapper.classify(story) == result + + def test_map_with_long_template(self, story): + """ + Tests `ValueError` is raised when result is too long. + """ + mapper = StorySlugMapper('paca' * 256) + + with pytest.raises(ValueError): + mapper(story) + + class TestMetaFormatMapper: """ MetaFormatMapper tests.