Merge remote-tracking branch 'origin/master' into redesign

This commit is contained in:
Luna D. 2024-12-02 21:18:05 +01:00
commit d7247b9031
No known key found for this signature in database
GPG key ID: 4B1C63448394F688
41 changed files with 1936 additions and 394 deletions

View file

@ -34,6 +34,9 @@ jobs:
- name: Build and test
run: docker compose run app run-test
- name: mix format
run: docker compose run app mix format --check-formatted
- name: Security lint
run: |
docker compose run app mix sobelow --config
@ -50,6 +53,24 @@ jobs:
- uses: actions/checkout@v4
- uses: crate-ci/typos@master
cargo:
name: Rust Linting and Unit Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: cargo fmt
run: (cd native/philomena && cargo fmt --check)
- name: cargo clippy
run: (cd native/philomena && cargo clippy -- -D warnings)
- name: cargo test
run: (cd native/philomena && cargo test)
lint-and-test:
name: 'JavaScript Linting and Unit Tests'
runs-on: ubuntu-latest
@ -80,4 +101,4 @@ jobs:
working-directory: ./assets
- run: npm run build
working-directory: ./assets
working-directory: ./assets

3
.gitignore vendored
View file

@ -43,8 +43,9 @@ npm-debug.log
# VS Code
.vscode
# ElixirLS
# Language server
.elixir_ls
.lexical
# Index dumps
*.jsonl

View file

@ -0,0 +1,159 @@
import { $, $$ } from '../utils/dom';
import { assertNotNull } from '../utils/assert';
import { setupQuickTag } from '../quick-tag';
import { fetchMock } from '../../test/fetch-mock.ts';
import { waitFor } from '@testing-library/dom';
const quickTagData = `<div>
<a class="js-quick-tag">Tag</a>
<a class="js-quick-tag--abort hidden"><span>Abort tagging</span></a>
<a class="js-quick-tag--submit hidden"><span>Submit</span></a>
<a class="js-quick-tag--all hidden"><span>Toggle all</span></a>
<div id="imagelist-container">
<div class="media-box" data-image-id="0">
<div class="media-box__header" data-image-id="0"></div>
</div>
<div class="media-box" data-image-id="1">
<div class="media-box__header" data-image-id="1"></div>
</div>
</div>
</div>`;
describe('Batch tagging', () => {
let tagButton: HTMLAnchorElement;
let abortButton: HTMLAnchorElement;
let submitButton: HTMLAnchorElement;
let toggleAllButton: HTMLAnchorElement;
let mediaBoxes: HTMLDivElement[];
beforeEach(() => {
localStorage.clear();
document.body.innerHTML = quickTagData;
tagButton = assertNotNull($<HTMLAnchorElement>('.js-quick-tag'));
abortButton = assertNotNull($<HTMLAnchorElement>('.js-quick-tag--abort'));
submitButton = assertNotNull($<HTMLAnchorElement>('.js-quick-tag--submit'));
toggleAllButton = assertNotNull($<HTMLAnchorElement>('.js-quick-tag--all'));
mediaBoxes = $$<HTMLDivElement>('.media-box');
});
afterEach(() => {
vi.restoreAllMocks();
});
it('should prompt the user on click', () => {
const spy = vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
tagButton.click();
expect(spy).toHaveBeenCalledOnce();
expect(tagButton.classList).toContain('hidden');
expect(abortButton.classList).not.toContain('hidden');
expect(submitButton.classList).not.toContain('hidden');
expect(toggleAllButton.classList).not.toContain('hidden');
});
it('should not modify media boxes before entry', () => {
mediaBoxes[0].click();
expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
});
it('should restore the list of tagged images on reload', () => {
// TODO: this is less than ideal, because it depends on the internal
// implementation of the quick-tag file. But we can't reload the page
// with jsdom.
localStorage.setItem('quickTagQueue', JSON.stringify(['0', '1']));
localStorage.setItem('quickTagName', JSON.stringify('a'));
setupQuickTag();
expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected');
expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected');
});
describe('after entry', () => {
beforeEach(() => {
vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
tagButton.click();
});
it('should abort the tagging process if accepted', () => {
const spy = vi.spyOn(window, 'confirm').mockImplementation(() => true);
abortButton.click();
expect(spy).toHaveBeenCalledOnce();
expect(tagButton.classList).not.toContain('hidden');
expect(abortButton.classList).toContain('hidden');
expect(submitButton.classList).toContain('hidden');
expect(toggleAllButton.classList).toContain('hidden');
});
it('should not abort the tagging process if rejected', () => {
const spy = vi.spyOn(window, 'confirm').mockImplementation(() => false);
abortButton.click();
expect(spy).toHaveBeenCalledOnce();
expect(tagButton.classList).toContain('hidden');
expect(abortButton.classList).not.toContain('hidden');
expect(submitButton.classList).not.toContain('hidden');
expect(toggleAllButton.classList).not.toContain('hidden');
});
it('should toggle media box state on click', () => {
mediaBoxes[0].click();
expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected');
expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
});
it('should toggle all media box states', () => {
mediaBoxes[0].click();
toggleAllButton.click();
expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected');
});
});
describe('for submission', () => {
beforeAll(() => {
fetchMock.enableMocks();
});
afterAll(() => {
fetchMock.disableMocks();
});
beforeEach(() => {
vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
tagButton.click();
fetchMock.resetMocks();
mediaBoxes[0].click();
mediaBoxes[1].click();
});
it('should return to normal state on successful submission', () => {
fetchMock.mockResponse('{"failed":[]}');
submitButton.click();
expect(fetch).toHaveBeenCalledOnce();
return waitFor(() => {
expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
});
});
it('should show error on failed submission', () => {
fetchMock.mockResponse('{"failed":[0,1]}');
submitButton.click();
const spy = vi.spyOn(window, 'alert').mockImplementation(() => {});
expect(fetch).toHaveBeenCalledOnce();
return waitFor(() => {
expect(spy).toHaveBeenCalledOnce();
expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
});
});
});
});

View file

@ -0,0 +1,99 @@
import { $ } from '../utils/dom';
import { assertNotNull } from '../utils/assert';
import { setupSearch } from '../search';
import { setupTagListener } from '../tagsinput';
const formData = `<form class="js-search-form">
<input type="text" class="js-search-field">
<a data-search-prepend="-">NOT</a>
<a data-search-add="id.lte:10" data-search-select-last="2" data-search-show-help="numeric">Numeric ID</a>
<a data-search-add="my:faves" data-search-show-help=" ">My favorites</a>
<div class="hidden" data-search-help="boolean">
<span class="js-search-help-subject"></span> is a Boolean value field
</div>
<div class="hidden" data-search-help="numeric">
<span class="js-search-help-subject"></span> is a numerical range field
</div>
</form>`;
describe('Search form help', () => {
beforeAll(() => {
setupSearch();
setupTagListener();
});
let input: HTMLInputElement;
let prependAnchor: HTMLAnchorElement;
let idAnchor: HTMLAnchorElement;
let favesAnchor: HTMLAnchorElement;
let helpNumeric: HTMLDivElement;
let subjectSpan: HTMLElement;
beforeEach(() => {
document.body.innerHTML = formData;
input = assertNotNull($<HTMLInputElement>('input'));
prependAnchor = assertNotNull($<HTMLAnchorElement>('a[data-search-prepend]'));
idAnchor = assertNotNull($<HTMLAnchorElement>('a[data-search-add="id.lte:10"]'));
favesAnchor = assertNotNull($<HTMLAnchorElement>('a[data-search-add="my:faves"]'));
helpNumeric = assertNotNull($<HTMLDivElement>('[data-search-help="numeric"]'));
subjectSpan = assertNotNull($<HTMLSpanElement>('span', helpNumeric));
});
it('should add text to input field', () => {
idAnchor.click();
expect(input.value).toBe('id.lte:10');
favesAnchor.click();
expect(input.value).toBe('id.lte:10, my:faves');
});
it('should focus and select text in input field when requested', () => {
idAnchor.click();
expect(input).toHaveFocus();
expect(input.selectionStart).toBe(7);
expect(input.selectionEnd).toBe(9);
});
it('should highlight subject name when requested', () => {
expect(helpNumeric).toHaveClass('hidden');
idAnchor.click();
expect(helpNumeric).not.toHaveClass('hidden');
expect(subjectSpan).toHaveTextContent('Numeric ID');
});
it('should not focus and select text in input field when unavailable', () => {
favesAnchor.click();
expect(input).not.toHaveFocus();
expect(input.selectionStart).toBe(8);
expect(input.selectionEnd).toBe(8);
});
it('should not highlight subject name when unavailable', () => {
favesAnchor.click();
expect(helpNumeric).toHaveClass('hidden');
});
it('should prepend to empty input', () => {
prependAnchor.click();
expect(input.value).toBe('-');
});
it('should prepend to single input', () => {
input.value = 'a';
prependAnchor.click();
expect(input.value).toBe('-a');
});
it('should prepend to comma-separated input', () => {
input.value = 'a,b';
prependAnchor.click();
expect(input.value).toBe('a,-b');
});
it('should prepend to comma and space-separated input', () => {
input.value = 'a, b';
prependAnchor.click();
expect(input.value).toBe('a, -b');
});
});

View file

@ -0,0 +1,188 @@
import { $, $$, hideEl } from '../utils/dom';
import { assertNotNull } from '../utils/assert';
import { TermSuggestion } from '../utils/suggestions';
import { setupTagsInput, addTag, reloadTagsInput } from '../tagsinput';
const formData = `<form class="tags-form">
<div class="js-tag-block fancy-tag-upload">
<textarea class="js-taginput js-taginput-plain"></textarea>
<div class="js-taginput js-taginput-fancy">
<input type="text" class="js-taginput-input" placeholder="add a tag">
</div>
</div>
<button class="js-taginput-show">Fancy Editor</button>
<button class="js-taginput-hide hidden">Plain Editor</button>
<input type="submit" value="Save Tags">
</form>`;
describe('Fancy tags input', () => {
let form: HTMLFormElement;
let tagBlock: HTMLDivElement;
let plainInput: HTMLTextAreaElement;
let fancyInput: HTMLDivElement;
let fancyText: HTMLInputElement;
let fancyShowButton: HTMLButtonElement;
let plainShowButton: HTMLButtonElement;
beforeEach(() => {
window.booru.fancyTagUpload = true;
window.booru.fancyTagEdit = true;
document.body.innerHTML = formData;
form = assertNotNull($<HTMLFormElement>('.tags-form'));
tagBlock = assertNotNull($<HTMLDivElement>('.js-tag-block'));
plainInput = assertNotNull($<HTMLTextAreaElement>('.js-taginput-plain'));
fancyInput = assertNotNull($<HTMLDivElement>('.js-taginput-fancy'));
fancyText = assertNotNull($<HTMLInputElement>('.js-taginput-input'));
fancyShowButton = assertNotNull($<HTMLButtonElement>('.js-taginput-show'));
plainShowButton = assertNotNull($<HTMLButtonElement>('.js-taginput-hide'));
// prevent these from submitting the form
fancyShowButton.addEventListener('click', e => e.preventDefault());
plainShowButton.addEventListener('click', e => e.preventDefault());
});
for (let i = 0; i < 4; i++) {
const type = (i & 2) === 0 ? 'upload' : 'edit';
const name = (i & 2) === 0 ? 'fancyTagUpload' : 'fancyTagEdit';
const value = (i & 1) === 0;
// eslint-disable-next-line no-loop-func
it(`should imply ${name}:${value} <-> ${type}:${value} on setup`, () => {
window.booru.fancyTagEdit = false;
window.booru.fancyTagUpload = false;
window.booru[name] = value;
plainInput.value = 'a, b';
tagBlock.classList.remove('fancy-tag-edit', 'fancy-tag-upload');
tagBlock.classList.add(`fancy-tag-${type}`);
expect($$('span.tag', fancyInput)).toHaveLength(0);
setupTagsInput(tagBlock);
expect($$('span.tag', fancyInput)).toHaveLength(value ? 2 : 0);
});
}
it('should move tags from the plain to the fancy editor when the fancy editor is shown', () => {
expect($$('span.tag', fancyInput)).toHaveLength(0);
setupTagsInput(tagBlock);
plainInput.value = 'a, b';
fancyShowButton.click();
expect($$('span.tag', fancyInput)).toHaveLength(2);
});
it('should move tags from the plain to the fancy editor on reload event', () => {
expect($$('span.tag', fancyInput)).toHaveLength(0);
setupTagsInput(tagBlock);
plainInput.value = 'a, b';
reloadTagsInput(plainInput);
expect($$('span.tag', fancyInput)).toHaveLength(2);
});
it('should respond to addtag events', () => {
setupTagsInput(tagBlock);
addTag(plainInput, 'a');
expect($$('span.tag', fancyInput)).toHaveLength(1);
});
it('should not respond to addtag events if the container is hidden', () => {
setupTagsInput(tagBlock);
hideEl(fancyInput);
addTag(plainInput, 'a');
expect($$('span.tag', fancyInput)).toHaveLength(0);
});
it('should respond to autocomplete events', () => {
setupTagsInput(tagBlock);
fancyText.dispatchEvent(new CustomEvent<TermSuggestion>('autocomplete', { detail: { value: 'a', label: 'a' } }));
expect($$('span.tag', fancyInput)).toHaveLength(1);
});
it('should allow removing previously added tags by clicking them', () => {
setupTagsInput(tagBlock);
addTag(plainInput, 'a');
assertNotNull($<HTMLAnchorElement>('span.tag a', fancyInput)).click();
expect($$('span.tag', fancyInput)).toHaveLength(0);
});
it('should allow removing previously added tags by adding one with a minus sign prepended', () => {
setupTagsInput(tagBlock);
addTag(plainInput, 'a');
expect($$('span.tag', fancyInput)).toHaveLength(1);
addTag(plainInput, '-a');
expect($$('span.tag', fancyInput)).toHaveLength(0);
});
it('should disallow adding empty tags', () => {
setupTagsInput(tagBlock);
addTag(plainInput, '');
expect($$('span.tag', fancyInput)).toHaveLength(0);
});
it('should disallow adding existing tags', () => {
setupTagsInput(tagBlock);
addTag(plainInput, 'a');
addTag(plainInput, 'a');
expect($$('span.tag', fancyInput)).toHaveLength(1);
});
it('should submit the form on ctrl+enter', () => {
setupTagsInput(tagBlock);
const ev = new KeyboardEvent('keydown', { keyCode: 13, ctrlKey: true, bubbles: true });
return new Promise<void>(resolve => {
form.addEventListener('submit', e => {
e.preventDefault();
resolve();
});
fancyText.dispatchEvent(ev);
expect(ev.defaultPrevented).toBe(true);
});
});
it('does nothing when backspacing on empty input and there are no tags', () => {
setupTagsInput(tagBlock);
const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true });
fancyText.dispatchEvent(ev);
expect($$('span.tag', fancyInput)).toHaveLength(0);
});
it('erases the last added tag when backspacing on empty input', () => {
setupTagsInput(tagBlock);
addTag(plainInput, 'a');
addTag(plainInput, 'b');
const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true });
fancyText.dispatchEvent(ev);
expect($$('span.tag', fancyInput)).toHaveLength(1);
});
it('adds new tag when comma is pressed', () => {
setupTagsInput(tagBlock);
const ev = new KeyboardEvent('keydown', { keyCode: 188, bubbles: true });
fancyText.value = 'a';
fancyText.dispatchEvent(ev);
expect($$('span.tag', fancyInput)).toHaveLength(1);
expect(fancyText.value).toBe('');
});
it('adds new tag when enter is pressed', () => {
setupTagsInput(tagBlock);
const ev = new KeyboardEvent('keydown', { keyCode: 13, bubbles: true });
fancyText.value = 'a';
fancyText.dispatchEvent(ev);
expect($$('span.tag', fancyInput)).toHaveLength(1);
expect(fancyText.value).toBe('');
});
});

View file

@ -25,6 +25,14 @@ function restoreOriginalValue() {
if (isSearchField(inputField) && originalQuery) {
inputField.value = originalQuery;
if (selectedTerm) {
const [, selectedTermEnd] = selectedTerm[0];
inputField.setSelectionRange(selectedTermEnd, selectedTermEnd);
}
return;
}
if (originalTerm) {

View file

@ -60,7 +60,7 @@ const markdownSyntax: Record<string, SyntaxHandler> = {
},
subscript: {
action: wrapSelection,
options: { prefix: '%' },
options: { prefix: '~' },
},
quote: {
action: wrapLines,

View file

@ -1,117 +0,0 @@
/**
* Quick Tag
*/
import store from './utils/store';
import { $, $$, toggleEl, onLeftClick } from './utils/dom';
import { fetchJson, handleError } from './utils/requests';
const imageQueueStorage = 'quickTagQueue';
const currentTagStorage = 'quickTagName';
function currentQueue() {
return store.get(imageQueueStorage) || [];
}
function currentTags() {
return store.get(currentTagStorage) || '';
}
function getTagButton() {
return $('.js-quick-tag');
}
function setTagButton(text) {
$('.js-quick-tag--submit span').textContent = text;
}
function toggleActiveState() {
toggleEl($('.js-quick-tag'), $('.js-quick-tag--abort'), $('.js-quick-tag--all'), $('.js-quick-tag--submit'));
setTagButton(`Submit (${currentTags()})`);
$$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected'));
$$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected'));
currentQueue().forEach(id =>
$$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')),
);
}
function activate() {
store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:'));
if (currentTags()) toggleActiveState();
}
function reset() {
store.remove(currentTagStorage);
store.remove(imageQueueStorage);
toggleActiveState();
}
function promptReset() {
if (window.confirm('Are you sure you want to abort batch tagging?')) {
reset();
}
}
function submit() {
setTagButton(`Wait... (${currentTags()})`);
fetchJson('PUT', '/admin/batch/tags', {
tags: currentTags(),
image_ids: currentQueue(),
})
.then(handleError)
.then(r => r.json())
.then(data => {
if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`);
reset();
});
}
function modifyImageQueue(mediaBox) {
if (currentTags()) {
const imageId = mediaBox.dataset.imageId;
const queue = currentQueue();
const isSelected = queue.includes(imageId);
isSelected ? queue.splice(queue.indexOf(imageId), 1) : queue.push(imageId);
$$(`.media-box__header[data-image-id="${imageId}"]`).forEach(el =>
el.classList.toggle('media-box__header--selected'),
);
store.set(imageQueueStorage, queue);
}
}
function toggleAllImages() {
$$('#imagelist-container .media-box').forEach(modifyImageQueue);
}
function clickHandler(event) {
const targets = {
'.js-quick-tag': activate,
'.js-quick-tag--abort': promptReset,
'.js-quick-tag--submit': submit,
'.js-quick-tag--all': toggleAllImages,
'.media-box': modifyImageQueue,
};
for (const target in targets) {
if (event.target && event.target.closest(target)) {
targets[target](event.target.closest(target));
currentTags() && event.preventDefault();
}
}
}
function setupQuickTag() {
if (getTagButton() && currentTags()) toggleActiveState();
if (getTagButton()) onLeftClick(clickHandler);
}
export { setupQuickTag };

124
assets/js/quick-tag.ts Normal file
View file

@ -0,0 +1,124 @@
/**
* Quick Tag
*/
import store from './utils/store';
import { assertNotNull, assertNotUndefined } from './utils/assert';
import { $, $$, toggleEl } from './utils/dom';
import { fetchJson, handleError } from './utils/requests';
import { delegate, leftClick } from './utils/events';
const imageQueueStorage = 'quickTagQueue';
const currentTagStorage = 'quickTagName';
function currentQueue(): string[] {
return store.get<string[]>(imageQueueStorage) || [];
}
function currentTags(): string {
return store.get<string>(currentTagStorage) || '';
}
function setTagButton(text: string) {
assertNotNull($('.js-quick-tag--submit span')).textContent = text;
}
function toggleActiveState() {
toggleEl($$<HTMLElement>('.js-quick-tag,.js-quick-tag--abort,.js-quick-tag--all,.js-quick-tag--submit'));
setTagButton(`Submit (${currentTags()})`);
$$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected'));
$$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected'));
currentQueue().forEach(id =>
$$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')),
);
}
function activate(event: Event) {
event.preventDefault();
store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:'));
if (currentTags()) {
toggleActiveState();
}
}
function reset() {
store.remove(currentTagStorage);
store.remove(imageQueueStorage);
toggleActiveState();
}
function promptReset(event: Event) {
event.preventDefault();
if (window.confirm('Are you sure you want to abort batch tagging?')) {
reset();
}
}
function submit(event: Event) {
event.preventDefault();
setTagButton(`Wait... (${currentTags()})`);
fetchJson('PUT', '/admin/batch/tags', {
tags: currentTags(),
image_ids: currentQueue(),
})
.then(handleError)
.then(r => r.json())
.then(data => {
if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`);
reset();
});
}
function modifyImageQueue(event: Event, mediaBox: HTMLDivElement) {
if (!currentTags()) {
return;
}
const imageId = assertNotUndefined(mediaBox.dataset.imageId);
const queue = currentQueue();
const isSelected = queue.includes(imageId);
if (isSelected) {
queue.splice(queue.indexOf(imageId), 1);
} else {
queue.push(imageId);
}
for (const boxHeader of $$(`.media-box__header[data-image-id="${imageId}"]`)) {
boxHeader.classList.toggle('media-box__header--selected');
}
store.set(imageQueueStorage, queue);
event.preventDefault();
}
function toggleAllImages(event: Event, _target: Element) {
for (const mediaBox of $$<HTMLDivElement>('#imagelist-container .media-box')) {
modifyImageQueue(event, mediaBox);
}
}
delegate(document, 'click', {
'.js-quick-tag': leftClick(activate),
'.js-quick-tag--abort': leftClick(promptReset),
'.js-quick-tag--submit': leftClick(submit),
'.js-quick-tag--all': leftClick(toggleAllImages),
'.media-box': leftClick(modifyImageQueue),
});
export function setupQuickTag() {
const tagButton = $<HTMLAnchorElement>('.js-quick-tag');
if (tagButton && currentTags()) {
toggleActiveState();
}
}

View file

@ -1,54 +1,85 @@
import { $, $$ } from './utils/dom';
import { assertNotNull, assertNotUndefined } from './utils/assert';
import { $, $$, showEl, hideEl } from './utils/dom';
import { delegate, leftClick } from './utils/events';
import { addTag } from './tagsinput';
function showHelp(subject: string, type: string | null) {
$$<HTMLElement>('[data-search-help]').forEach(helpBox => {
if (helpBox.getAttribute('data-search-help') === type) {
const searchSubject = $<HTMLElement>('.js-search-help-subject', helpBox);
if (searchSubject) {
searchSubject.textContent = subject;
}
helpBox.classList.remove('hidden');
} else {
helpBox.classList.add('hidden');
}
});
}
function prependToLast(field: HTMLInputElement, value: string) {
const separatorIndex = field.value.lastIndexOf(',');
const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1;
field.value =
field.value.slice(0, separatorIndex + advanceBy) + value + field.value.slice(separatorIndex + advanceBy);
}
function selectLast(field: HTMLInputElement, characterCount: number) {
function focusAndSelectLast(field: HTMLInputElement, characterCount: number) {
field.focus();
field.selectionStart = field.value.length - characterCount;
field.selectionEnd = field.value.length;
}
function executeFormHelper(e: PointerEvent) {
if (!e.target) {
return;
}
function prependToLast(field: HTMLInputElement, value: string) {
// Find the last comma in the input and advance past it
const separatorIndex = field.value.lastIndexOf(',');
const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1;
const searchField = $<HTMLInputElement>('.js-search-field');
const attr = (name: string) => e.target && (e.target as HTMLElement).getAttribute(name);
// Insert the value string at the new location
field.value = [
field.value.slice(0, separatorIndex + advanceBy),
value,
field.value.slice(separatorIndex + advanceBy),
].join('');
}
if (attr('data-search-add')) addTag(searchField, attr('data-search-add'));
if (attr('data-search-show-help')) showHelp((e.target as Node).textContent || '', attr('data-search-show-help'));
if (attr('data-search-select-last') && searchField) {
selectLast(searchField, parseInt(attr('data-search-select-last') || '', 10));
function getAssociatedData(target: HTMLElement) {
const form = assertNotNull(target.closest('form'));
const input = assertNotNull($<HTMLInputElement>('.js-search-field', form));
const helpBoxes = $$<HTMLDivElement>('[data-search-help]', form);
return { input, helpBoxes };
}
function showHelp(helpBoxes: HTMLDivElement[], typeName: string, subject: string) {
for (const helpBox of helpBoxes) {
// Get the subject name span
const subjectName = assertNotNull($<HTMLElement>('.js-search-help-subject', helpBox));
// Take the appropriate action for this help box
if (helpBox.dataset.searchHelp === typeName) {
subjectName.textContent = subject;
showEl(helpBox);
} else {
hideEl(helpBox);
}
}
if (attr('data-search-prepend') && searchField) prependToLast(searchField, attr('data-search-prepend') || '');
}
function onSearchAdd(_event: Event, target: HTMLAnchorElement) {
// Load form
const { input, helpBoxes } = getAssociatedData(target);
// Get data for this link
const addValue = assertNotUndefined(target.dataset.searchAdd);
const showHelpValue = assertNotUndefined(target.dataset.searchShowHelp);
const selectLastValue = target.dataset.searchSelectLast;
// Add the tag
addTag(input, addValue);
// Show associated help, if available
showHelp(helpBoxes, showHelpValue, assertNotNull(target.textContent));
// Select last characters, if requested
if (selectLastValue) {
focusAndSelectLast(input, Number(selectLastValue));
}
}
function onSearchPrepend(_event: Event, target: HTMLAnchorElement) {
// Load form
const { input } = getAssociatedData(target);
// Get data for this link
const prependValue = assertNotUndefined(target.dataset.searchPrepend);
// Prepend
prependToLast(input, prependValue);
}
export function setupSearch() {
const form = $<HTMLInputElement>('.js-search-form');
if (form) form.addEventListener('click', executeFormHelper as EventListener);
delegate(document, 'click', {
'form.js-search-form a[data-search-add][data-search-show-help]': leftClick(onSearchAdd),
'form.js-search-form a[data-search-prepend]': leftClick(onSearchPrepend),
});
}

View file

@ -2,14 +2,20 @@
* Fancy tag editor.
*/
import { assertNotNull, assertType } from './utils/assert';
import { $, $$, clearEl, removeEl, showEl, hideEl, escapeCss, escapeHtml } from './utils/dom';
import { TermSuggestion } from './utils/suggestions';
function setupTagsInput(tagBlock) {
const [textarea, container] = $$('.js-taginput', tagBlock);
const setup = $('.js-tag-block ~ button', tagBlock.parentNode);
const inputField = $('input', container);
export function setupTagsInput(tagBlock: HTMLDivElement) {
const form = assertNotNull(tagBlock.closest('form'));
const textarea = assertNotNull($<HTMLTextAreaElement>('.js-taginput-plain', tagBlock));
const container = assertNotNull($<HTMLDivElement>('.js-taginput-fancy', tagBlock));
const parentField = assertNotNull(tagBlock.parentElement);
const setup = assertNotNull($<HTMLButtonElement>('.js-tag-block ~ button', parentField));
const inputField = assertNotNull($<HTMLInputElement>('input', container));
const submitButton = assertNotNull($<HTMLInputElement | HTMLButtonElement>('[type="submit"]', form));
let tags = [];
let tags: string[] = [];
// Load in the current tag set from the textarea
setup.addEventListener('click', importTags);
@ -27,7 +33,7 @@ function setupTagsInput(tagBlock) {
inputField.addEventListener('keydown', handleKeyEvent);
// Respond to autocomplete form clicks
inputField.addEventListener('autocomplete', handleAutocomplete);
inputField.addEventListener('autocomplete', handleAutocomplete as EventListener);
// Respond to Ctrl+Enter shortcut
tagBlock.addEventListener('keydown', handleCtrlEnter);
@ -35,19 +41,19 @@ function setupTagsInput(tagBlock) {
// TODO: Cleanup this bug fix
// Switch to fancy tagging if user settings want it
if (fancyEditorRequested(tagBlock)) {
showEl($$('.js-taginput-fancy'));
showEl($$('.js-taginput-hide'));
hideEl($$('.js-taginput-plain'));
hideEl($$('.js-taginput-show'));
showEl($$<HTMLElement>('.js-taginput-fancy'));
showEl($$<HTMLElement>('.js-taginput-hide'));
hideEl($$<HTMLElement>('.js-taginput-plain'));
hideEl($$<HTMLElement>('.js-taginput-show'));
importTags();
}
function handleAutocomplete(event) {
function handleAutocomplete(event: CustomEvent<TermSuggestion>) {
insertTag(event.detail.value);
inputField.focus();
}
function handleAddTag(event) {
function handleAddTag(event: AddtagEvent) {
// Ignore if not in tag edit mode
if (container.classList.contains('hidden')) return;
@ -55,14 +61,16 @@ function setupTagsInput(tagBlock) {
event.stopPropagation();
}
function handleTagClear(event) {
if (event.target.dataset.tagName) {
function handleTagClear(event: Event) {
const target = assertType(event.target, HTMLElement);
if (target.dataset.tagName) {
event.preventDefault();
removeTag(event.target.dataset.tagName, event.target.parentNode);
removeTag(target.dataset.tagName, assertNotNull(target.parentElement));
}
}
function handleKeyEvent(event) {
function handleKeyEvent(event: KeyboardEvent) {
const { keyCode, ctrlKey, shiftKey } = event;
// allow form submission with ctrl+enter if no text was typed
@ -73,7 +81,7 @@ function setupTagsInput(tagBlock) {
// backspace on a blank input field
if (keyCode === 8 && inputField.value === '') {
event.preventDefault();
const erased = $('.tag:last-of-type', container);
const erased = $<HTMLElement>('.tag:last-of-type', container);
if (erased) removeTag(tags[tags.length - 1], erased);
}
@ -86,14 +94,14 @@ function setupTagsInput(tagBlock) {
}
}
function handleCtrlEnter(event) {
function handleCtrlEnter(event: KeyboardEvent) {
const { keyCode, ctrlKey } = event;
if (keyCode !== 13 || !ctrlKey) return;
$('[type="submit"]', tagBlock.closest('form')).click();
submitButton.click();
}
function insertTag(name) {
function insertTag(name: string) {
name = name.trim(); // eslint-disable-line no-param-reassign
// Add if not degenerate or already present
@ -102,9 +110,9 @@ function setupTagsInput(tagBlock) {
// Remove instead if the tag name starts with a minus
if (name[0] === '-') {
name = name.slice(1); // eslint-disable-line no-param-reassign
const tagLink = $(`[data-tag-name="${escapeCss(name)}"]`, container);
const tagLink = assertNotNull($(`[data-tag-name="${escapeCss(name)}"]`, container));
return removeTag(name, tagLink.parentNode);
return removeTag(name, assertNotNull(tagLink.parentElement));
}
tags.push(name);
@ -116,7 +124,7 @@ function setupTagsInput(tagBlock) {
inputField.value = '';
}
function removeTag(name, element) {
function removeTag(name: string, element: HTMLElement) {
removeEl(element);
// Remove the tag from the list
@ -134,7 +142,7 @@ function setupTagsInput(tagBlock) {
}
}
function fancyEditorRequested(tagBlock) {
function fancyEditorRequested(tagBlock: HTMLDivElement) {
// Check whether the user made the fancy editor the default for each type of tag block.
return (
(window.booru.fancyTagUpload && tagBlock.classList.contains('fancy-tag-upload')) ||
@ -142,19 +150,17 @@ function fancyEditorRequested(tagBlock) {
);
}
function setupTagListener() {
export function setupTagListener() {
document.addEventListener('addtag', event => {
if (event.target.value) event.target.value += ', ';
event.target.value += event.detail.name;
});
}
function addTag(textarea, name) {
export function addTag(textarea: HTMLInputElement | HTMLTextAreaElement, name: string) {
textarea.dispatchEvent(new CustomEvent('addtag', { detail: { name }, bubbles: true }));
}
function reloadTagsInput(textarea) {
export function reloadTagsInput(textarea: HTMLInputElement | HTMLTextAreaElement) {
textarea.dispatchEvent(new CustomEvent('reload'));
}
export { setupTagsInput, setupTagListener, addTag, reloadTagsInput };

View file

@ -2380,9 +2380,9 @@
}
},
"node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",

View file

@ -7,6 +7,8 @@ import { fireEvent } from '@testing-library/dom';
window.booru = {
timeAgo: () => {},
csrfToken: 'mockCsrfToken',
fancyTagEdit: true,
fancyTagUpload: true,
hiddenTag: '/mock-tagblocked.svg',
hiddenTagList: [],
hideStaffTools: 'true',

View file

@ -73,6 +73,14 @@ interface BooruObject {
* List of image IDs in the current gallery.
*/
galleryImages?: number[];
/**
* Fancy tag setting for uploading images.
*/
fancyTagUpload: boolean;
/**
* Fancy tag setting for editing images.
*/
fancyTagEdit: boolean;
}
declare global {

20
assets/types/tags.ts Normal file
View file

@ -0,0 +1,20 @@
export {};
declare global {
interface Addtag {
name: string;
}
interface AddtagEvent extends CustomEvent<Addtag> {
target: HTMLInputElement | HTMLTextAreaElement;
}
interface ReloadEvent extends CustomEvent {
target: HTMLInputElement | HTMLTextAreaElement;
}
interface GlobalEventHandlersEventMap {
addtag: AddtagEvent;
reload: ReloadEvent;
}
}

View file

@ -64,16 +64,14 @@ services:
- POSTGRES_PASSWORD=postgres
volumes:
- postgres_data:/var/lib/postgresql/data
logging:
driver: "none"
attach: false
opensearch:
image: opensearchproject/opensearch:2.16.0
volumes:
- opensearch_data:/usr/share/opensearch/data
- ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
logging:
driver: "none"
attach: false
ulimits:
nofile:
soft: 65536
@ -81,8 +79,7 @@ services:
valkey:
image: valkey/valkey:8.0-alpine
logging:
driver: "none"
attach: false
files:
image: andrewgaul/s3proxy:sha-4976e17
@ -90,6 +87,7 @@ services:
- JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3
volumes:
- .:/srv/philomena
attach: false
web:
build:
@ -106,8 +104,7 @@ services:
environment:
- AWS_ACCESS_KEY_ID=local-identity
- AWS_SECRET_ACCESS_KEY=local-credential
logging:
driver: "none"
attach: false
depends_on:
- app
ports:

View file

@ -121,7 +121,9 @@ defmodule Mix.Tasks.UploadToS3 do
end
defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
queryable
|> Batch.record_batches(batch_size: batch_size)
|> Enum.each(fn models ->
models
|> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name),
timeout: :infinity
@ -142,7 +144,9 @@ defmodule Mix.Tasks.UploadToS3 do
end
defp upload_images(queryable, batch_size, file_root, new_file_root) do
Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
queryable
|> Batch.record_batches(batch_size: batch_size)
|> Enum.each(fn models ->
models
|> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity)
|> Stream.run()

View file

@ -0,0 +1,171 @@
defmodule Philomena.DataExports.Aggregator do
@moduledoc """
Data generation module for data export logic.
"""
import Ecto.Query
alias PhilomenaQuery.Batch
# Direct PII
alias Philomena.Donations.Donation
alias Philomena.UserFingerprints.UserFingerprint
alias Philomena.UserIps.UserIp
alias Philomena.UserNameChanges.UserNameChange
alias Philomena.Users.User
# UGC for export
alias Philomena.ArtistLinks.ArtistLink
alias Philomena.Badges.Award
alias Philomena.Comments.Comment
alias Philomena.Commissions.Commission
alias Philomena.DnpEntries.DnpEntry
alias Philomena.DuplicateReports.DuplicateReport
alias Philomena.Filters.Filter
alias Philomena.ImageFaves.ImageFave
alias Philomena.ImageHides.ImageHide
alias Philomena.ImageVotes.ImageVote
alias Philomena.Images.Image
alias Philomena.PollVotes.PollVote
alias Philomena.Posts.Post
alias Philomena.Reports.Report
alias Philomena.SourceChanges.SourceChange
alias Philomena.TagChanges.TagChange
alias Philomena.Topics.Topic
alias Philomena.Bans.User, as: UserBan
# Direct UGC from form submission
@user_columns [
:created_at,
:name,
:email,
:description,
:current_filter_id,
:spoiler_type,
:theme,
:images_per_page,
:show_large_thumbnails,
:show_sidebar_and_watched_images,
:fancy_tag_field_on_upload,
:fancy_tag_field_on_edit,
:fancy_tag_field_in_settings,
:autorefresh_by_default,
:anonymous_by_default,
:comments_newest_first,
:comments_always_jump_to_last,
:comments_per_page,
:watch_on_reply,
:watch_on_new_topic,
:watch_on_upload,
:messages_newest_first,
:serve_webm,
:no_spoilered_in_watched,
:watched_images_query_str,
:watched_images_exclude_str,
:use_centered_layout,
:personal_title,
:hide_vote_counts,
:scale_large_images
]
# All these also have created_at and are selected by user_id
@indirect_columns [
{Donation, [:email, :amount, :fee, :note]},
{UserFingerprint, [:fingerprint, :uses, :updated_at]},
{UserIp, [:ip, :uses, :updated_at]},
{UserNameChange, [:name]},
{ArtistLink, [:aasm_state, :uri, :public, :tag_id]},
{Award, [:label, :badge_name, :badge_id]},
{Comment,
[
:ip,
:fingerprint,
:user_agent,
:referrer,
:anonymous,
:image_id,
:edited_at,
:edit_reason,
:body
]},
{Commission,
[:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]},
{DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions],
:requesting_user_id},
{DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]},
{Filter,
[
:name,
:description,
:public,
:hidden_complex_str,
:spoilered_complex_str,
:hidden_tag_ids,
:spoilered_tag_ids
]},
{ImageFave, [:image_id], :user_id, :image_id},
{ImageHide, [:image_id], :user_id, :image_id},
{ImageVote, [:image_id, :up], :user_id, :image_id},
{Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]},
{PollVote, [:rank, :poll_option_id]},
{Post,
[:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]},
{Report,
[:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]},
{SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]},
{TagChange,
[:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]},
{Topic, [:title, :anonymous, :forum_id]},
{UserBan, [:reason, :generated_ban_id]}
]
@doc """
Get all of the export data for the given user.
"""
def get_for_user(user_id) do
[select_user(user_id)] ++ select_indirect(user_id)
end
defp select_user(user_id) do
select_schema_by_key(user_id, User, @user_columns, :id)
end
defp select_indirect(user_id) do
Enum.map(@indirect_columns, fn
{schema_name, columns} ->
select_schema_by_key(user_id, schema_name, columns)
{schema_name, columns, key_column} ->
select_schema_by_key(user_id, schema_name, columns, key_column)
{schema_name, columns, key_column, id_field} ->
select_schema_by_key(user_id, schema_name, columns, key_column, id_field)
end)
end
defp select_schema_by_key(
user_id,
schema_name,
columns,
key_column \\ :user_id,
id_field \\ :id
) do
table_name = schema_name.__schema__(:source)
columns = [:created_at] ++ columns
{"#{table_name}.jsonl",
schema_name
|> where([s], field(s, ^key_column) == ^user_id)
|> select([s], map(s, ^columns))
|> Batch.records(id_field: id_field)
|> results_as_json_lines()}
end
defp results_as_json_lines(list_of_maps) do
Stream.map(list_of_maps, fn map ->
map
|> Map.new(fn {k, v} -> {k, to_string(v)} end)
|> Jason.encode!()
|> Kernel.<>("\n")
end)
end
end

View file

@ -0,0 +1,56 @@
defmodule Philomena.DataExports.ZipGenerator do
@moduledoc """
ZIP file generator for an export.
"""
alias Philomena.Native
@doc """
Write the ZIP file for the given aggregate data.
Expects a list of 2-tuples, with the first element being the name of the
file to generate, and the second element being a stream which generates the
binary contents of the file.
"""
@spec generate(Path.t(), Enumerable.t()) :: :ok | atom()
def generate(filename, aggregate) do
case Native.zip_open_writer(filename) do
{:ok, zip} ->
stream_aggregate(zip, aggregate)
error ->
error
end
end
@spec stream_aggregate(reference(), Enumerable.t()) :: {:ok, reference()} | :error
defp stream_aggregate(zip, aggregate) do
aggregate
|> Enum.reduce_while(:ok, fn {name, content_stream}, _ ->
with :ok <- Native.zip_start_file(zip, name),
:ok <- stream_file_data(zip, content_stream) do
{:cont, :ok}
else
error ->
{:halt, error}
end
end)
|> case do
:ok ->
Native.zip_finish(zip)
error ->
error
end
end
@spec stream_file_data(reference(), Enumerable.t(iodata())) :: :ok | :error
defp stream_file_data(zip, content_stream) do
Enum.reduce_while(content_stream, :ok, fn iodata, _ ->
case Native.zip_write(zip, IO.iodata_to_binary(iodata)) do
:ok -> {:cont, :ok}
error -> {:halt, error}
end
end)
end
end

View file

@ -11,4 +11,16 @@ defmodule Philomena.Native do
@spec camo_image_url(String.t()) :: String.t()
def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
@spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()}
def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded)
@spec zip_start_file(reference(), String.t()) :: :ok | :error
def zip_start_file(_zip, _name), do: :erlang.nif_error(:nif_not_loaded)
@spec zip_write(reference(), binary()) :: :ok | :error
def zip_write(_zip, _data), do: :erlang.nif_error(:nif_not_loaded)
@spec zip_finish(reference()) :: :ok | :error
def zip_finish(_zip), do: :erlang.nif_error(:nif_not_loaded)
end

View file

@ -15,7 +15,8 @@ defmodule Philomena.UserDownvoteWipe do
ImageVote
|> where(user_id: ^user.id, up: false)
|> Batch.query_batches([id_field: :image_id], fn queryable ->
|> Batch.query_batches(id_field: :image_id)
|> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id))
{count, nil} =
@ -31,7 +32,8 @@ defmodule Philomena.UserDownvoteWipe do
if upvotes_and_faves_too do
ImageVote
|> where(user_id: ^user.id, up: true)
|> Batch.query_batches([id_field: :image_id], fn queryable ->
|> Batch.query_batches(id_field: :image_id)
|> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id))
{count, nil} =
@ -46,7 +48,8 @@ defmodule Philomena.UserDownvoteWipe do
ImageFave
|> where(user_id: ^user.id)
|> Batch.query_batches([id_field: :image_id], fn queryable ->
|> Batch.query_batches(id_field: :image_id)
|> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_f], i_f.image_id))
{count, nil} =

View file

@ -27,7 +27,9 @@ defmodule Philomena.TagChangeRevertWorker do
batch_size = attributes["batch_size"] || 100
attributes = Map.delete(attributes, "batch_size")
Batch.query_batches(queryable, [batch_size: batch_size], fn queryable ->
queryable
|> Batch.query_batches(batch_size: batch_size)
|> Enum.each(fn queryable ->
ids = Repo.all(select(queryable, [tc], tc.id))
TagChanges.mass_revert(ids, cast_ip(atomify_keys(attributes)))
end)

View file

@ -21,6 +21,7 @@ defmodule PhilomenaProxy.Scrapers do
}
@scrapers [
PhilomenaProxy.Scrapers.Bluesky,
PhilomenaProxy.Scrapers.Deviantart,
PhilomenaProxy.Scrapers.Pillowfort,
PhilomenaProxy.Scrapers.Twitter,

View file

@ -0,0 +1,76 @@
defmodule PhilomenaProxy.Scrapers.Bluesky do
@moduledoc false
alias PhilomenaProxy.Scrapers.Scraper
alias PhilomenaProxy.Scrapers
@behaviour Scraper
@url_regex ~r|https://bsky\.app/profile/([^/]+)/post/([^/?#]+)|
@fullsize_image_regex ~r|.*/img/feed_fullsize/plain/([^/]+)/([^@]+).*|
@blob_image_url_pattern "https://bsky.social/xrpc/com.atproto.sync.getBlob/?did=\\1&cid=\\2"
@spec can_handle?(URI.t(), String.t()) :: boolean()
def can_handle?(_uri, url) do
String.match?(url, @url_regex)
end
@spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result()
def scrape(_uri, url) do
[handle, id] = Regex.run(@url_regex, url, capture: :all_but_first)
did = fetch_did(handle)
api_url_get_posts =
"https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}"
post_json =
api_url_get_posts
|> PhilomenaProxy.Http.get()
|> json!()
|> Map.fetch!("posts")
|> hd()
%{
source_url: url,
author_name: domain_first_component(post_json["author"]["handle"]),
description: post_json["record"]["text"],
images:
Enum.map(
post_json["embed"]["images"],
&%{
url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern),
camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"])
}
)
}
end
defp fetch_did(handle) do
case handle do
<<"did:", _rest::binary>> ->
handle
_ ->
api_url_resolve_handle =
"https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}"
api_url_resolve_handle
|> PhilomenaProxy.Http.get()
|> json!()
|> Map.fetch!("did")
end
end
defp domain_first_component(domain) do
case String.split(domain, ".") do
[name | _] ->
name
_ ->
domain
end
end
defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body)
end

View file

@ -25,24 +25,32 @@ defmodule PhilomenaQuery.Batch do
@type id_field :: {:id_field, atom()}
@type batch_options :: [batch_size() | id_field()]
@typedoc """
The callback for `record_batches/3`.
@doc """
Stream schema structures on a queryable, using batches to avoid locking.
Valid options:
* `batch_size` (integer) - the number of records to load per batch
* `id_field` (atom) - the name of the field containing the ID
## Example
queryable = from i in Image, where: i.image_width >= 1920
queryable
|> PhilomenaQuery.Batch.record_batches()
|> Enum.each(fn image -> IO.inspect(image.id) end)
Takes a list of schema structs which were returned in the batch. Return value is ignored.
"""
@type record_batch_callback :: ([struct()] -> any())
@typedoc """
The callback for `query_batches/3`.
Takes an `m:Ecto.Query` that can be processed with `m:Philomena.Repo` query commands, such
as `Philomena.Repo.update_all/3` or `Philomena.Repo.delete_all/2`. Return value is ignored.
"""
@type query_batch_callback :: ([Ecto.Query.t()] -> any())
@spec records(queryable(), batch_options()) :: Enumerable.t()
def records(queryable, opts \\ []) do
queryable
|> query_batches(opts)
|> Stream.flat_map(&Repo.all/1)
end
@doc """
Execute a callback with lists of schema structures on a queryable,
using batches to avoid locking.
Stream lists of schema structures on a queryable, using batches to avoid
locking.
Valid options:
* `batch_size` (integer) - the number of records to load per batch
@ -56,16 +64,20 @@ defmodule PhilomenaQuery.Batch do
Enum.each(images, &IO.inspect(&1.id))
end
PhilomenaQuery.Batch.record_batches(queryable, cb)
queryable
|> PhilomenaQuery.Batch.record_batches()
|> Enum.each(cb)
"""
@spec record_batches(queryable(), batch_options(), record_batch_callback()) :: []
def record_batches(queryable, opts \\ [], callback) do
query_batches(queryable, opts, &callback.(Repo.all(&1)))
@spec record_batches(queryable(), batch_options()) :: Enumerable.t()
def record_batches(queryable, opts \\ []) do
queryable
|> query_batches(opts)
|> Stream.map(&Repo.all/1)
end
@doc """
Execute a callback with bulk queries on a queryable, using batches to avoid locking.
Stream bulk queries on a queryable, using batches to avoid locking.
Valid options:
* `batch_size` (integer) - the number of records to load per batch
@ -76,41 +88,36 @@ defmodule PhilomenaQuery.Batch do
> If you are looking to receive schema structures (e.g., you are querying for `Image`s,
> and you want to receive `Image` objects, then use `record_batches/3` instead.
An `m:Ecto.Query` which selects all IDs in the current batch is passed into the callback
during each invocation.
`m:Ecto.Query` structs which select the IDs in each batch are streamed out.
## Example
queryable = from ui in ImageVote, where: ui.user_id == 1234
opts = [id_field: :image_id]
cb = fn bulk_query ->
Repo.delete_all(bulk_query)
end
PhilomenaQuery.Batch.query_batches(queryable, opts, cb)
queryable
|> PhilomenaQuery.Batch.query_batches(id_field: :image_id)
|> Enum.each(fn batch_query -> Repo.delete_all(batch_query) end)
"""
@spec query_batches(queryable(), batch_options(), query_batch_callback()) :: []
def query_batches(queryable, opts \\ [], callback) do
ids = load_ids(queryable, -1, opts)
query_batches(queryable, opts, callback, ids)
end
defp query_batches(_queryable, _opts, _callback, []), do: []
defp query_batches(queryable, opts, callback, ids) do
@spec query_batches(queryable(), batch_options()) :: Enumerable.t(Ecto.Query.t())
def query_batches(queryable, opts \\ []) do
id_field = Keyword.get(opts, :id_field, :id)
queryable
|> where([m], field(m, ^id_field) in ^ids)
|> callback.()
Stream.unfold(
load_ids(queryable, -1, opts),
fn
[] ->
# Stop when no more results are produced
nil
ids = load_ids(queryable, Enum.max(ids), opts)
ids ->
# Process results and output next query
output = where(queryable, [m], field(m, ^id_field) in ^ids)
next_ids = load_ids(queryable, Enum.max(ids), opts)
query_batches(queryable, opts, callback, ids)
{output, next_ids}
end
)
end
defp load_ids(queryable, max_id, opts) do
@ -118,8 +125,9 @@ defmodule PhilomenaQuery.Batch do
batch_size = Keyword.get(opts, :batch_size, 1000)
queryable
|> exclude(:preload)
|> exclude(:order_by)
|> exclude(:preload)
|> exclude(:select)
|> order_by(asc: ^id_field)
|> where([m], field(m, ^id_field) > ^max_id)
|> select([m], field(m, ^id_field))

View file

@ -199,11 +199,13 @@ defmodule PhilomenaQuery.Search do
Search.reindex(query, Image, batch_size: 5000)
"""
@spec reindex(queryable(), schema_module(), Batch.batch_options()) :: []
@spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok
def reindex(queryable, module, opts \\ []) do
index = @policy.index_for(module)
Batch.record_batches(queryable, opts, fn records ->
queryable
|> Batch.record_batches(opts)
|> Enum.each(fn records ->
lines =
Enum.flat_map(records, fn record ->
doc = index.as_json(record)

View file

@ -4,6 +4,7 @@ defmodule PhilomenaWeb.Image.SourceController do
alias Philomena.SourceChanges.SourceChange
alias Philomena.UserStatistics
alias Philomena.Images.Image
alias Philomena.Images.Source
alias Philomena.Images
alias Philomena.Repo
import Ecto.Query
@ -41,7 +42,9 @@ defmodule PhilomenaWeb.Image.SourceController do
PhilomenaWeb.Api.Json.ImageView.render("show.json", %{image: image, interactions: []})
)
changeset = Images.change_image(image)
changeset =
%{image | sources: sources_for_edit(image.sources)}
|> Images.change_image()
source_change_count =
SourceChange
@ -74,4 +77,9 @@ defmodule PhilomenaWeb.Image.SourceController do
)
end
end
# TODO: this is duplicated in ImageController
defp sources_for_edit(), do: [%Source{}]
defp sources_for_edit([]), do: sources_for_edit()
defp sources_for_edit(sources), do: sources
end

View file

@ -219,6 +219,7 @@ defmodule PhilomenaWeb.ImageController do
end
end
# TODO: this is duplicated in Image.SourceController
defp sources_for_edit(), do: [%Source{}]
defp sources_for_edit([]), do: sources_for_edit()
defp sources_for_edit(sources), do: sources

View file

@ -75,8 +75,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do
defp render_representations(images, conn) do
loaded_images = load_images(images)
images
|> Enum.map(fn group ->
Map.new(images, fn group ->
img = loaded_images[Enum.at(group, 0)]
text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}"
@ -131,8 +130,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do
|> Phoenix.HTML.Safe.to_iodata()
|> IO.iodata_to_binary()
[text, string_contents]
{text, string_contents}
end)
|> Map.new(fn [id, html] -> {id, html} end)
end
end

View file

@ -62,3 +62,6 @@ elixir:
- true ->
p We couldn't find any commission listings to display. Sorry!
.block__header.page__header
.page__pagination = pagination

View file

@ -1,7 +1,7 @@
h1 Search
= form_for :search, ~p"/search", [id: "searchform", method: "get", class: "js-search-form", enforce_utf8: false], fn f ->
= text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"]
= text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"], data: [ac: "true", ac_min_length: 3, ac_mode: "search"]
.block
.block__header.flex

View file

@ -77,7 +77,7 @@ defmodule Philomena.MixProject do
{:mail, "~> 0.3.0"},
# Markdown
{:rustler, "~> 0.27"},
{:rustler, "~> 0.35"},
# Linting
{:credo, "~> 1.6", only: [:dev, :test], runtime: false},

View file

@ -5,14 +5,14 @@
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"canada": {:hex, :canada, "1.0.2", "040e4c47609b0a67d5773ac1fbe5e99f840cef173d69b739beda7c98453e0770", [:mix], [], "hexpm", "4269f74153fe89583fe50bd4d5de57bfe01f31258a6b676d296f3681f1483c68"},
"canary": {:git, "https://github.com/marcinkoziej/canary.git", "704debde7a2c0600f78c687807884bf37c45bd79", [ref: "704debde7a2c0600f78c687807884bf37c45bd79"]},
"castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"},
"castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"},
"credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"},
"credo_envvar": {:hex, :credo_envvar, "0.1.4", "40817c10334e400f031012c0510bfa0d8725c19d867e4ae39cf14f2cbebc3b20", [:mix], [{:credo, "~> 1.0", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "5055cdb4bcbaf7d423bc2bb3ac62b4e2d825e2b1e816884c468dee59d0363009"},
"credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
"db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
"decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"},
"dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
"earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"},
"ecto": {:hex, :ecto, "3.12.1", "626765f7066589de6fa09e0876a253ff60c3d00870dd3a1cd696e2ba67bfceea", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df0045ab9d87be947228e05a8d153f3e06e0d05ab10c3b3cc557d2f7243d1940"},
@ -27,7 +27,7 @@
"expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"},
"exq": {:hex, :exq, "0.19.0", "06eb92944dad39f0954dc8f63190d3e24d11734eef88cf5800883e57ebf74f3c", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 6.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "24fc0ebdd87cc7406e1034fb46c2419f9c8a362f0ec634d23b6b819514d36390"},
"file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
"finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
"finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"},
"gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"},
"hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
"inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"},
@ -62,8 +62,8 @@
"qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"},
"redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"},
"remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"},
"req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"},
"rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"},
"req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
"rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"},
"scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"},
"scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]},
"secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"},
@ -71,7 +71,7 @@
"sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"},
"sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
"swoosh": {:hex, :swoosh, "1.16.10", "04be6e2eb1a31aa0aa21a731175c81cc3998189456a92daf13d44a5c754afcf5", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "756be04db173c0cbe318f1dfe2bcc88aa63aed78cf5a4b02b61b36ee11fc716a"},
"telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
"telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
"thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"},
"toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},

View file

@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "adler2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
[[package]]
name = "aho-corasick"
version = "1.1.3"
@ -11,12 +17,46 @@ dependencies = [
"memchr",
]
[[package]]
name = "arbitrary"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
dependencies = [
"derive_arbitrary",
]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "bon"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509"
dependencies = [
"bon-macros",
"rustversion",
]
[[package]]
name = "bon-macros"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33"
dependencies = [
"darling",
"ident_case",
"prettyplease",
"proc-macro2",
"quote",
"rustversion",
"syn",
]
[[package]]
name = "bumpalo"
version = "3.16.0"
@ -29,6 +69,16 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
[[package]]
name = "caseless"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
dependencies = [
"regex",
"unicode-normalization",
]
[[package]]
name = "cc"
version = "1.0.94"
@ -43,12 +93,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "comrak"
version = "0.24.1"
source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a"
version = "0.29.0"
source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.2#00ac2a12d5797feb0ceba9a98487451ab65593fe"
dependencies = [
"derive_builder",
"bon",
"caseless",
"entities",
"http",
"memchr",
"once_cell",
"regex",
@ -58,10 +108,25 @@ dependencies = [
]
[[package]]
name = "darling"
version = "0.20.9"
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "darling"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
dependencies = [
"darling_core",
"darling_macro",
@ -69,9 +134,9 @@ dependencies = [
[[package]]
name = "darling_core"
version = "0.20.9"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
dependencies = [
"fnv",
"ident_case",
@ -83,9 +148,9 @@ dependencies = [
[[package]]
name = "darling_macro"
version = "0.20.9"
version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
@ -93,48 +158,55 @@ dependencies = [
]
[[package]]
name = "derive_builder"
version = "0.20.0"
name = "derive_arbitrary"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "derive_builder_macro"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
dependencies = [
"derive_builder_core",
"syn",
]
[[package]]
name = "deunicode"
version = "1.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "322ef0094744e63628e6f0eb2295517f79276a5b342a4c2ff3042566ca181d4e"
[[package]]
name = "displaydoc"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "entities"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca"
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "flate2"
version = "1.0.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -151,10 +223,16 @@ dependencies = [
]
[[package]]
name = "heck"
version = "0.4.1"
name = "hashbrown"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "http"
@ -183,6 +261,22 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
dependencies = [
"equivalent",
"hashbrown",
]
[[package]]
name = "inventory"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
[[package]]
name = "itoa"
version = "1.0.11"
@ -218,18 +312,28 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "libloading"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
dependencies = [
"cfg-if",
"windows-targets",
]
[[package]]
name = "lockfree-object-pool"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
[[package]]
name = "log"
version = "0.4.21"
@ -238,15 +342,24 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "memchr"
version = "2.7.2"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "miniz_oxide"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
dependencies = [
"adler2",
]
[[package]]
name = "once_cell"
version = "1.19.0"
version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
[[package]]
name = "percent-encoding"
@ -260,17 +373,31 @@ version = "0.3.0"
dependencies = [
"base64",
"comrak",
"http",
"jemallocator",
"once_cell",
"regex",
"ring",
"rustler",
"url",
"zip",
]
[[package]]
name = "prettyplease"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.80"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e"
checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
dependencies = [
"unicode-ident",
]
@ -307,6 +434,12 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-lite"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
[[package]]
name = "regex-syntax"
version = "0.8.3"
@ -330,36 +463,40 @@ dependencies = [
[[package]]
name = "rustler"
version = "0.28.0"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d7a2f98cb272ae0548e434bb3afde626012084cbebef84542bc7afed868bd6f"
checksum = "b705f2c3643cc170d8888cb6bad589155d9c0248f3104ef7a04c2b7ffbaf13fc"
dependencies = [
"lazy_static",
"inventory",
"libloading",
"regex-lite",
"rustler_codegen",
"rustler_sys",
]
[[package]]
name = "rustler_codegen"
version = "0.28.0"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ae37fb154683a1ff13e95dfd16c15043d6aee9853ea650ca34a45bd94e319a5"
checksum = "3ad56caff00562948bd6ac33c18dbc579e5a1bbee2d7f2f54073307e57f6b57a"
dependencies = [
"heck",
"inventory",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "rustler_sys"
version = "2.2.1"
name = "rustversion"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae0eb19e2fdf4abc3662441087fc499a58b7550822cc8007e79f15c40157c883"
dependencies = [
"regex",
"unreachable",
]
checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
[[package]]
name = "simd-adler32"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]]
name = "slug"
@ -385,15 +522,35 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.59"
version = "2.0.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a"
checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
@ -442,15 +599,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unreachable"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
dependencies = [
"void",
]
[[package]]
name = "untrusted"
version = "0.7.1"
@ -468,12 +616,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
@ -559,3 +701,98 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "zip"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494"
dependencies = [
"arbitrary",
"crc32fast",
"crossbeam-utils",
"displaydoc",
"flate2",
"indexmap",
"memchr",
"thiserror",
"zopfli",
]
[[package]]
name = "zopfli"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
dependencies = [
"bumpalo",
"crc32fast",
"lockfree-object-pool",
"log",
"once_cell",
"simd-adler32",
]

View file

@ -10,12 +10,16 @@ path = "src/lib.rs"
crate-type = ["dylib"]
[dependencies]
comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false }
jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
rustler = "0.28"
ring = "0.16"
base64 = "0.21"
comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false }
http = "0.2"
jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
once_cell = "1.20"
regex = "1"
ring = "0.16"
rustler = "0.35"
url = "2.3"
zip = { version = "2.2.0", features = ["deflate"], default-features = false }
[profile.release]
opt-level = 3

View file

@ -1,8 +1,8 @@
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64::Engine;
use ring::hmac;
use std::env;
use url::Url;
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
fn trusted_host(mut url: Url) -> Option<String> {
url.set_port(Some(443)).ok()?;
@ -11,7 +11,7 @@ fn trusted_host(mut url: Url) -> Option<String> {
Some(url.to_string())
}
fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option<String> {
fn untrusted_host(url: Url, camo_host: &str, camo_key: &str) -> Option<String> {
let camo_url = format!("https://{}", camo_host);
let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref());
let tag = hmac::sign(&key, url.to_string().as_bytes());
@ -27,20 +27,24 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option<Strin
Some(camo_uri.to_string())
}
pub fn image_url(uri: String) -> Option<String> {
pub fn image_url(uri: &str) -> Option<String> {
let cdn_host = env::var("CDN_HOST").ok()?;
let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| String::from(""));
let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| String::from(""));
let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| "".into());
let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| "".into());
if camo_key.is_empty() {
return Some(uri);
return Some(uri.into());
}
let url = Url::parse(&uri).ok()?;
let url = Url::parse(uri).ok()?;
match url.host_str() {
Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url),
Some(_) => untrusted_host(url, camo_host, camo_key),
None => Some(String::from("")),
Some(_) => untrusted_host(url, &camo_host, &camo_key),
None => Some("".into()),
}
}
pub fn image_url_careful(uri: &str) -> String {
image_url(uri).unwrap_or_else(|| "".into())
}

View file

@ -0,0 +1,38 @@
use std::collections::BTreeSet;
use std::env;
use http::Uri;
use once_cell::sync::Lazy;
use regex::Regex;
pub type DomainSet = BTreeSet<String>;
static DOMAINS: Lazy<Option<DomainSet>> = Lazy::new(|| {
if let Ok(domains) = env::var("SITE_DOMAINS") {
return Some(domains.split(',').map(|s| s.to_string()).collect());
}
None
});
pub fn get() -> &'static Option<DomainSet> {
&DOMAINS
}
pub fn relativize(domains: &DomainSet, url: &str) -> Option<String> {
let uri = url.parse::<Uri>().ok()?;
if let Some(a) = uri.authority() {
if domains.contains(a.host()) {
if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) {
return Some(re.replace(url, "").into());
}
}
}
Some(url.into())
}
pub fn relativize_careful(domains: &DomainSet, url: &str) -> String {
relativize(domains, url).unwrap_or_else(|| url.into())
}

View file

@ -1,32 +1,58 @@
use jemallocator::Jemalloc;
use rustler::Term;
use rustler::{Atom, Binary};
use std::collections::HashMap;
mod camo;
mod domains;
mod markdown;
#[cfg(test)]
mod tests;
mod zip;
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
rustler::init! {
"Elixir.Philomena.Native",
[markdown_to_html, markdown_to_html_unsafe, camo_image_url]
"Elixir.Philomena.Native"
}
// Markdown NIF wrappers.
#[rustler::nif(schedule = "DirtyCpu")]
fn markdown_to_html(input: String, reps: Term) -> String {
fn markdown_to_html(input: &str, reps: HashMap<String, String>) -> String {
markdown::to_html(input, reps)
}
#[rustler::nif(schedule = "DirtyCpu")]
fn markdown_to_html_unsafe(input: String, reps: Term) -> String {
fn markdown_to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String {
markdown::to_html_unsafe(input, reps)
}
// Camo NIF wrappers.
#[rustler::nif]
fn camo_image_url(input: String) -> String {
camo::image_url(input).unwrap_or_else(|| String::from(""))
fn camo_image_url(input: &str) -> String {
camo::image_url_careful(input)
}
// Zip NIF wrappers.
#[rustler::nif]
fn zip_open_writer(path: &str) -> Result<zip::WriterResourceArc, Atom> {
zip::open_writer(path)
}
#[rustler::nif]
fn zip_start_file(writer: zip::WriterResourceArc, name: &str) -> Atom {
zip::start_file(writer, name)
}
#[rustler::nif(schedule = "DirtyCpu")]
fn zip_write(writer: zip::WriterResourceArc, data: Binary) -> Atom {
zip::write(writer, data.as_slice())
}
#[rustler::nif(schedule = "DirtyCpu")]
fn zip_finish(writer: zip::WriterResourceArc) -> Atom {
zip::finish(writer)
}

View file

@ -1,53 +1,54 @@
use comrak::ComrakOptions;
use crate::camo;
use rustler::{MapIterator, Term};
use crate::{camo, domains};
use comrak::Options;
use std::collections::HashMap;
use std::env;
use std::sync::Arc;
fn common_options() -> ComrakOptions {
let mut options = ComrakOptions::default();
pub fn common_options() -> Options {
let mut options = Options::default();
// Upstream options
options.extension.autolink = true;
options.extension.table = true;
options.extension.description_lists = true;
options.extension.superscript = true;
options.extension.strikethrough = true;
options.extension.philomena = true;
options.parse.smart = true;
options.render.hardbreaks = true;
options.render.github_pre_lang = true;
options.render.escape = true;
options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from("")));
// Philomena options
options.extension.underline = true;
options.extension.spoiler = true;
options.extension.greentext = true;
options.extension.subscript = true;
options.extension.philomena = true;
options.render.ignore_empty_links = true;
options.render.ignore_setext = true;
if let Ok(domains) = env::var("SITE_DOMAINS") {
options.extension.philomena_domains = Some(domains.split(',').map(|s| s.to_string()).collect::<Vec<String>>());
options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url)));
if let Some(domains) = domains::get() {
options.extension.link_url_rewriter = Some(Arc::new(|url: &str| {
domains::relativize_careful(domains, url)
}));
}
options
}
fn map_to_hashmap(map: Term) -> Option<HashMap<String, String>> {
Some(MapIterator::new(map)?.map(|(key, value)| {
let key: String = key.decode().unwrap_or_else(|_| String::from(""));
let value: String = value.decode().unwrap_or_else(|_| String::from(""));
pub fn to_html(input: &str, reps: HashMap<String, String>) -> String {
let mut options = common_options();
options.extension.replacements = Some(reps);
(key, value)
}).collect())
comrak::markdown_to_html(input, &options)
}
pub fn to_html(input: String, reps: Term) -> String {
let mut options = common_options();
options.render.escape = true;
options.extension.philomena_replacements = map_to_hashmap(reps);
comrak::markdown_to_html(&input, &options)
}
pub fn to_html_unsafe(input: String, reps: Term) -> String {
pub fn to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String {
let mut options = common_options();
options.render.escape = false;
options.render.unsafe_ = true;
options.extension.replacements = Some(reps);
options.extension.philomena_replacements = map_to_hashmap(reps);
comrak::markdown_to_html(&input, &options)
comrak::markdown_to_html(input, &options)
}

View file

@ -0,0 +1,273 @@
use std::{collections::HashMap, sync::Arc};
use crate::{domains, markdown::*};
fn test_options() -> comrak::Options {
let mut options = common_options();
options.extension.image_url_rewriter = None;
options.extension.link_url_rewriter = None;
options
}
fn html(input: &str, expected: &str) {
html_opts_w(input, expected, &test_options());
}
fn html_opts_i<F>(input: &str, expected: &str, opts: F)
where
F: Fn(&mut comrak::Options),
{
let mut options = test_options();
opts(&mut options);
html_opts_w(input, expected, &options);
}
fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) {
let output = comrak::markdown_to_html(input, options);
if output != expected {
println!("Input:");
println!("========================");
println!("{}", input);
println!("========================");
println!("Expected:");
println!("========================");
println!("{}", expected);
println!("========================");
println!("Output:");
println!("========================");
println!("{}", output);
println!("========================");
}
assert_eq!(output, expected);
}
#[test]
fn subscript() {
html("H~2~O\n", "<div class=\"paragraph\">H<sub>2</sub>O</div>\n");
}
#[test]
fn subscript_autolink_interaction() {
html(
"https://example.com/search?q=1%2C2%2C3",
"<div class=\"paragraph\"><a href=\"https://example.com/search?q=1%2C2%2C3\">https://example.com/search?q=1%2C2%2C3</a></div>\n"
);
}
#[test]
fn underscore_autolink_interaction() {
html(
"https://example.com/x_",
"<div class=\"paragraph\"><a href=\"https://example.com/x_\">https://example.com/x_</a></div>\n"
)
}
#[test]
fn spoiler() {
html(
"The ||dog dies at the end of Marley and Me||.\n",
"<div class=\"paragraph\">The <span class=\"spoiler\">dog dies at the end of Marley and Me</span>.</div>\n",
);
}
#[test]
fn spoiler_in_table() {
html(
"Text | Result\n--- | ---\n`||some clever text||` | ||some clever text||\n",
concat!(
"<table>\n",
"<thead>\n",
"<tr>\n",
"<th>Text</th>\n",
"<th>Result</th>\n",
"</tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr>\n",
"<td><code>||some clever text||</code></td>\n",
"<td><span class=\"spoiler\">some clever text</span></td>\n",
"</tr>\n",
"</tbody>\n",
"</table>\n"
),
);
}
#[test]
fn spoiler_regressions() {
html(
"|should not be spoiler|\n||should be spoiler||\n|||should be spoiler surrounded by pipes|||",
concat!(
"<div class=\"paragraph\">|should not be spoiler|<br />\n",
"<span class=\"spoiler\">should be spoiler</span><br />\n",
"|<span class=\"spoiler\">should be spoiler surrounded by pipes</span>|</div>\n"
),
);
}
#[test]
fn mismatched_spoilers() {
html(
"|||this is a spoiler with pipe in front||\n||this is not a spoiler|\n||this is a spoiler with pipe after|||",
concat!(
"<div class=\"paragraph\">|<span class=\"spoiler\">this is a spoiler with pipe in front</span><br />\n",
"||this is not a spoiler|<br />\n",
"<span class=\"spoiler\">this is a spoiler with pipe after</span>|</div>\n"
),
);
}
#[test]
fn underline() {
html(
"__underlined__\n",
"<div class=\"paragraph\"><u>underlined</u></div>\n",
);
}
#[test]
fn no_setext_headings_in_philomena() {
html(
"text text\n---",
"<div class=\"paragraph\">text text</div>\n<hr />\n",
);
}
#[test]
fn greentext_preserved() {
html(
">implying\n>>implying",
"<div class=\"paragraph\">&gt;implying<br />\n&gt;&gt;implying</div>\n",
);
}
#[test]
fn separate_quotes_on_line_end() {
html(
"> 1\n>\n> 2",
"<blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n<div class=\"paragraph\">&gt;</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n"
);
}
#[test]
fn unnest_quotes_on_line_end() {
html(
"> 1\n> > 2\n> 1",
"<blockquote>\n<div class=\"paragraph\">1</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n",
);
}
#[test]
fn unnest_quotes_on_line_end_commonmark() {
html(
"> 1\n> > 2\n> \n> 1",
"<blockquote>\n<div class=\"paragraph\">1</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n",
);
}
#[test]
fn philomena_images() {
html(
"![full](http://example.com/image.png)",
"<div class=\"paragraph\"><span class=\"imgspoiler\"><img src=\"http://example.com/image.png\" alt=\"full\" /></span></div>\n",
);
}
#[test]
fn no_empty_link() {
html_opts_i(
"[](https://example.com/evil.domain.for.seo.spam)",
"<div class=\"paragraph\">[](https://example.com/evil.domain.for.seo.spam)</div>\n",
|opts| opts.extension.autolink = false,
);
html_opts_i(
"[ ](https://example.com/evil.domain.for.seo.spam)",
"<div class=\"paragraph\">[ ](https://example.com/evil.domain.for.seo.spam)</div>\n",
|opts| opts.extension.autolink = false,
);
}
#[test]
fn empty_image_allowed() {
html(
"![ ](https://example.com/evil.domain.for.seo.spam)",
"<div class=\"paragraph\"><span class=\"imgspoiler\"><img src=\"https://example.com/evil.domain.for.seo.spam\" alt=\" \" /></span></div>\n",
);
}
#[test]
fn image_inside_link_allowed() {
html(
"[![](https://example.com/image.png)](https://example.com/)",
"<div class=\"paragraph\"><a href=\"https://example.com/\"><span class=\"imgspoiler\"><img src=\"https://example.com/image.png\" alt=\"\" /></span></a></div>\n",
);
}
#[test]
fn image_mention() {
html_opts_i(
"hello world >>1234p >>1337",
"<div class=\"paragraph\">hello world <div id=\"1234\">p</div> &gt;&gt;1337</div>\n",
|opts| {
let mut replacements = HashMap::new();
replacements.insert("1234p".to_string(), "<div id=\"1234\">p</div>".to_string());
opts.extension.replacements = Some(replacements);
},
);
}
#[test]
fn image_mention_line_start() {
html_opts_i(
">>1234p",
"<div class=\"paragraph\"><div id=\"1234\">p</div></div>\n",
|opts| {
let mut replacements = HashMap::new();
replacements.insert("1234p".to_string(), "<div id=\"1234\">p</div>".to_string());
opts.extension.replacements = Some(replacements);
},
);
}
#[test]
fn auto_relative_links() {
let domains = Arc::new(vec!["example.com".into()].into_iter().collect());
let f = Arc::new(move |url: &str| domains::relativize_careful(&*domains, url));
html_opts_i(
"[some link text](https://example.com/some/path)",
"<div class=\"paragraph\"><a href=\"/some/path\">some link text</a></div>\n",
|opts| {
opts.extension.link_url_rewriter = Some(f.clone());
},
);
html_opts_i(
"https://example.com/some/path",
"<div class=\"paragraph\"><a href=\"/some/path\">https://example.com/some/path</a></div>\n",
|opts| {
opts.extension.link_url_rewriter = Some(f.clone());
},
);
html_opts_i(
"[some link text](https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345)",
"<div class=\"paragraph\"><a href=\"/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345\">some link text</a></div>\n",
|opts| {
opts.extension.link_url_rewriter = Some(f.clone());
},
);
html_opts_i(
"https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345",
"<div class=\"paragraph\"><a href=\"/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345\">https://example.com/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345</a></div>\n",
|opts| {
opts.extension.link_url_rewriter = Some(f.clone());
},
);
}

View file

@ -0,0 +1,67 @@
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::sync::Mutex;
use rustler::{Atom, Resource, ResourceArc};
use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter};
mod atoms {
rustler::atoms! {
ok,
error,
}
}
pub struct WriterResource {
inner: Mutex<Option<ZipWriter<File>>>,
}
#[rustler::resource_impl]
impl Resource for WriterResource {}
pub type WriterResourceArc = ResourceArc<WriterResource>;
fn with_writer<F, T>(writer: WriterResourceArc, f: F) -> Atom
where
F: FnOnce(&mut Option<ZipWriter<File>>) -> Option<T>,
{
let mut guard = match writer.inner.lock() {
Ok(g) => g,
Err(_) => return atoms::error(),
};
match f(&mut guard) {
Some(_) => atoms::ok(),
None => atoms::error(),
}
}
pub fn open_writer(path: &str) -> Result<WriterResourceArc, Atom> {
match OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(path)
{
Ok(file) => Ok(ResourceArc::new(WriterResource {
inner: Mutex::new(Some(ZipWriter::new(file))),
})),
Err(_) => Err(atoms::error()),
}
}
pub fn start_file(writer: WriterResourceArc, name: &str) -> Atom {
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
with_writer(writer, |writer| {
writer.as_mut()?.start_file(name, options).ok()
})
}
pub fn write(writer: WriterResourceArc, data: &[u8]) -> Atom {
with_writer(writer, |writer| writer.as_mut()?.write(data).ok())
}
pub fn finish(writer: WriterResourceArc) -> Atom {
with_writer(writer, |writer| writer.take().map(|w| w.finish().ok()))
}