diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml
index a92aa72f..066732f5 100644
--- a/.github/workflows/elixir.yml
+++ b/.github/workflows/elixir.yml
@@ -34,6 +34,9 @@ jobs:
- name: Build and test
run: docker compose run app run-test
+ - name: mix format
+ run: docker compose run app mix format --check-formatted
+
- name: Security lint
run: |
docker compose run app mix sobelow --config
@@ -50,6 +53,24 @@ jobs:
- uses: actions/checkout@v4
- uses: crate-ci/typos@master
+ cargo:
+ name: Rust Linting and Unit Tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: cargo fmt
+ run: (cd native/philomena && cargo fmt --check)
+
+ - name: cargo clippy
+ run: (cd native/philomena && cargo clippy -- -D warnings)
+
+ - name: cargo test
+ run: (cd native/philomena && cargo test)
+
lint-and-test:
name: 'JavaScript Linting and Unit Tests'
runs-on: ubuntu-latest
@@ -80,4 +101,4 @@ jobs:
working-directory: ./assets
- run: npm run build
- working-directory: ./assets
\ No newline at end of file
+ working-directory: ./assets
diff --git a/.gitignore b/.gitignore
index da2c70b6..0c221544 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,8 +43,9 @@ npm-debug.log
# VS Code
.vscode
-# ElixirLS
+# Language server
.elixir_ls
+.lexical
# Index dumps
*.jsonl
diff --git a/assets/js/__tests__/quick-tag.spec.ts b/assets/js/__tests__/quick-tag.spec.ts
new file mode 100644
index 00000000..6b5d1ecd
--- /dev/null
+++ b/assets/js/__tests__/quick-tag.spec.ts
@@ -0,0 +1,159 @@
+import { $, $$ } from '../utils/dom';
+import { assertNotNull } from '../utils/assert';
+import { setupQuickTag } from '../quick-tag';
+import { fetchMock } from '../../test/fetch-mock.ts';
+import { waitFor } from '@testing-library/dom';
+
+const quickTagData = `
`;
+
+describe('Batch tagging', () => {
+ let tagButton: HTMLAnchorElement;
+ let abortButton: HTMLAnchorElement;
+ let submitButton: HTMLAnchorElement;
+ let toggleAllButton: HTMLAnchorElement;
+ let mediaBoxes: HTMLDivElement[];
+
+ beforeEach(() => {
+ localStorage.clear();
+ document.body.innerHTML = quickTagData;
+
+ tagButton = assertNotNull($('.js-quick-tag'));
+ abortButton = assertNotNull($('.js-quick-tag--abort'));
+ submitButton = assertNotNull($('.js-quick-tag--submit'));
+ toggleAllButton = assertNotNull($('.js-quick-tag--all'));
+ mediaBoxes = $$('.media-box');
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('should prompt the user on click', () => {
+ const spy = vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
+ tagButton.click();
+
+ expect(spy).toHaveBeenCalledOnce();
+ expect(tagButton.classList).toContain('hidden');
+ expect(abortButton.classList).not.toContain('hidden');
+ expect(submitButton.classList).not.toContain('hidden');
+ expect(toggleAllButton.classList).not.toContain('hidden');
+ });
+
+ it('should not modify media boxes before entry', () => {
+ mediaBoxes[0].click();
+ expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
+ });
+
+ it('should restore the list of tagged images on reload', () => {
+ // TODO: this is less than ideal, because it depends on the internal
+ // implementation of the quick-tag file. But we can't reload the page
+ // with jsdom.
+ localStorage.setItem('quickTagQueue', JSON.stringify(['0', '1']));
+ localStorage.setItem('quickTagName', JSON.stringify('a'));
+
+ setupQuickTag();
+ expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected');
+ expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected');
+ });
+
+ describe('after entry', () => {
+ beforeEach(() => {
+ vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
+ tagButton.click();
+ });
+
+ it('should abort the tagging process if accepted', () => {
+ const spy = vi.spyOn(window, 'confirm').mockImplementation(() => true);
+ abortButton.click();
+
+ expect(spy).toHaveBeenCalledOnce();
+ expect(tagButton.classList).not.toContain('hidden');
+ expect(abortButton.classList).toContain('hidden');
+ expect(submitButton.classList).toContain('hidden');
+ expect(toggleAllButton.classList).toContain('hidden');
+ });
+
+ it('should not abort the tagging process if rejected', () => {
+ const spy = vi.spyOn(window, 'confirm').mockImplementation(() => false);
+ abortButton.click();
+
+ expect(spy).toHaveBeenCalledOnce();
+ expect(tagButton.classList).toContain('hidden');
+ expect(abortButton.classList).not.toContain('hidden');
+ expect(submitButton.classList).not.toContain('hidden');
+ expect(toggleAllButton.classList).not.toContain('hidden');
+ });
+
+ it('should toggle media box state on click', () => {
+ mediaBoxes[0].click();
+ expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected');
+ expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
+ });
+
+ it('should toggle all media box states', () => {
+ mediaBoxes[0].click();
+ toggleAllButton.click();
+ expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
+ expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected');
+ });
+ });
+
+ describe('for submission', () => {
+ beforeAll(() => {
+ fetchMock.enableMocks();
+ });
+
+ afterAll(() => {
+ fetchMock.disableMocks();
+ });
+
+ beforeEach(() => {
+ vi.spyOn(window, 'prompt').mockImplementation(() => 'a');
+ tagButton.click();
+
+ fetchMock.resetMocks();
+ mediaBoxes[0].click();
+ mediaBoxes[1].click();
+ });
+
+ it('should return to normal state on successful submission', () => {
+ fetchMock.mockResponse('{"failed":[]}');
+ submitButton.click();
+
+ expect(fetch).toHaveBeenCalledOnce();
+
+ return waitFor(() => {
+ expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
+ expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
+ });
+ });
+
+ it('should show error on failed submission', () => {
+ fetchMock.mockResponse('{"failed":[0,1]}');
+ submitButton.click();
+
+ const spy = vi.spyOn(window, 'alert').mockImplementation(() => {});
+
+ expect(fetch).toHaveBeenCalledOnce();
+
+ return waitFor(() => {
+ expect(spy).toHaveBeenCalledOnce();
+ expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected');
+ expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected');
+ });
+ });
+ });
+});
diff --git a/assets/js/__tests__/search.spec.ts b/assets/js/__tests__/search.spec.ts
new file mode 100644
index 00000000..abc033b5
--- /dev/null
+++ b/assets/js/__tests__/search.spec.ts
@@ -0,0 +1,99 @@
+import { $ } from '../utils/dom';
+import { assertNotNull } from '../utils/assert';
+import { setupSearch } from '../search';
+import { setupTagListener } from '../tagsinput';
+
+const formData = ``;
+
+describe('Search form help', () => {
+ beforeAll(() => {
+ setupSearch();
+ setupTagListener();
+ });
+
+ let input: HTMLInputElement;
+ let prependAnchor: HTMLAnchorElement;
+ let idAnchor: HTMLAnchorElement;
+ let favesAnchor: HTMLAnchorElement;
+ let helpNumeric: HTMLDivElement;
+ let subjectSpan: HTMLElement;
+
+ beforeEach(() => {
+ document.body.innerHTML = formData;
+
+ input = assertNotNull($('input'));
+ prependAnchor = assertNotNull($('a[data-search-prepend]'));
+ idAnchor = assertNotNull($('a[data-search-add="id.lte:10"]'));
+ favesAnchor = assertNotNull($('a[data-search-add="my:faves"]'));
+ helpNumeric = assertNotNull($('[data-search-help="numeric"]'));
+ subjectSpan = assertNotNull($('span', helpNumeric));
+ });
+
+ it('should add text to input field', () => {
+ idAnchor.click();
+ expect(input.value).toBe('id.lte:10');
+
+ favesAnchor.click();
+ expect(input.value).toBe('id.lte:10, my:faves');
+ });
+
+ it('should focus and select text in input field when requested', () => {
+ idAnchor.click();
+ expect(input).toHaveFocus();
+ expect(input.selectionStart).toBe(7);
+ expect(input.selectionEnd).toBe(9);
+ });
+
+ it('should highlight subject name when requested', () => {
+ expect(helpNumeric).toHaveClass('hidden');
+ idAnchor.click();
+ expect(helpNumeric).not.toHaveClass('hidden');
+ expect(subjectSpan).toHaveTextContent('Numeric ID');
+ });
+
+ it('should not focus and select text in input field when unavailable', () => {
+ favesAnchor.click();
+ expect(input).not.toHaveFocus();
+ expect(input.selectionStart).toBe(8);
+ expect(input.selectionEnd).toBe(8);
+ });
+
+ it('should not highlight subject name when unavailable', () => {
+ favesAnchor.click();
+ expect(helpNumeric).toHaveClass('hidden');
+ });
+
+ it('should prepend to empty input', () => {
+ prependAnchor.click();
+ expect(input.value).toBe('-');
+ });
+
+ it('should prepend to single input', () => {
+ input.value = 'a';
+ prependAnchor.click();
+ expect(input.value).toBe('-a');
+ });
+
+ it('should prepend to comma-separated input', () => {
+ input.value = 'a,b';
+ prependAnchor.click();
+ expect(input.value).toBe('a,-b');
+ });
+
+ it('should prepend to comma and space-separated input', () => {
+ input.value = 'a, b';
+ prependAnchor.click();
+ expect(input.value).toBe('a, -b');
+ });
+});
diff --git a/assets/js/__tests__/tagsinput.spec.ts b/assets/js/__tests__/tagsinput.spec.ts
new file mode 100644
index 00000000..e32b2dfc
--- /dev/null
+++ b/assets/js/__tests__/tagsinput.spec.ts
@@ -0,0 +1,188 @@
+import { $, $$, hideEl } from '../utils/dom';
+import { assertNotNull } from '../utils/assert';
+import { TermSuggestion } from '../utils/suggestions';
+import { setupTagsInput, addTag, reloadTagsInput } from '../tagsinput';
+
+const formData = ``;
+
+describe('Fancy tags input', () => {
+ let form: HTMLFormElement;
+ let tagBlock: HTMLDivElement;
+ let plainInput: HTMLTextAreaElement;
+ let fancyInput: HTMLDivElement;
+ let fancyText: HTMLInputElement;
+ let fancyShowButton: HTMLButtonElement;
+ let plainShowButton: HTMLButtonElement;
+
+ beforeEach(() => {
+ window.booru.fancyTagUpload = true;
+ window.booru.fancyTagEdit = true;
+ document.body.innerHTML = formData;
+
+ form = assertNotNull($('.tags-form'));
+ tagBlock = assertNotNull($('.js-tag-block'));
+ plainInput = assertNotNull($('.js-taginput-plain'));
+ fancyInput = assertNotNull($('.js-taginput-fancy'));
+ fancyText = assertNotNull($('.js-taginput-input'));
+ fancyShowButton = assertNotNull($('.js-taginput-show'));
+ plainShowButton = assertNotNull($('.js-taginput-hide'));
+
+ // prevent these from submitting the form
+ fancyShowButton.addEventListener('click', e => e.preventDefault());
+ plainShowButton.addEventListener('click', e => e.preventDefault());
+ });
+
+ for (let i = 0; i < 4; i++) {
+ const type = (i & 2) === 0 ? 'upload' : 'edit';
+ const name = (i & 2) === 0 ? 'fancyTagUpload' : 'fancyTagEdit';
+ const value = (i & 1) === 0;
+
+ // eslint-disable-next-line no-loop-func
+ it(`should imply ${name}:${value} <-> ${type}:${value} on setup`, () => {
+ window.booru.fancyTagEdit = false;
+ window.booru.fancyTagUpload = false;
+ window.booru[name] = value;
+
+ plainInput.value = 'a, b';
+ tagBlock.classList.remove('fancy-tag-edit', 'fancy-tag-upload');
+ tagBlock.classList.add(`fancy-tag-${type}`);
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+
+ setupTagsInput(tagBlock);
+ expect($$('span.tag', fancyInput)).toHaveLength(value ? 2 : 0);
+ });
+ }
+
+ it('should move tags from the plain to the fancy editor when the fancy editor is shown', () => {
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+
+ setupTagsInput(tagBlock);
+ plainInput.value = 'a, b';
+ fancyShowButton.click();
+ expect($$('span.tag', fancyInput)).toHaveLength(2);
+ });
+
+ it('should move tags from the plain to the fancy editor on reload event', () => {
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+
+ setupTagsInput(tagBlock);
+ plainInput.value = 'a, b';
+ reloadTagsInput(plainInput);
+ expect($$('span.tag', fancyInput)).toHaveLength(2);
+ });
+
+ it('should respond to addtag events', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, 'a');
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ });
+
+ it('should not respond to addtag events if the container is hidden', () => {
+ setupTagsInput(tagBlock);
+ hideEl(fancyInput);
+ addTag(plainInput, 'a');
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+ });
+
+ it('should respond to autocomplete events', () => {
+ setupTagsInput(tagBlock);
+ fancyText.dispatchEvent(new CustomEvent('autocomplete', { detail: { value: 'a', label: 'a' } }));
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ });
+
+ it('should allow removing previously added tags by clicking them', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, 'a');
+ assertNotNull($('span.tag a', fancyInput)).click();
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+ });
+
+ it('should allow removing previously added tags by adding one with a minus sign prepended', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, 'a');
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ addTag(plainInput, '-a');
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+ });
+
+ it('should disallow adding empty tags', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, '');
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+ });
+
+ it('should disallow adding existing tags', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, 'a');
+ addTag(plainInput, 'a');
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ });
+
+ it('should submit the form on ctrl+enter', () => {
+ setupTagsInput(tagBlock);
+
+ const ev = new KeyboardEvent('keydown', { keyCode: 13, ctrlKey: true, bubbles: true });
+
+ return new Promise(resolve => {
+ form.addEventListener('submit', e => {
+ e.preventDefault();
+ resolve();
+ });
+
+ fancyText.dispatchEvent(ev);
+ expect(ev.defaultPrevented).toBe(true);
+ });
+ });
+
+ it('does nothing when backspacing on empty input and there are no tags', () => {
+ setupTagsInput(tagBlock);
+
+ const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true });
+ fancyText.dispatchEvent(ev);
+
+ expect($$('span.tag', fancyInput)).toHaveLength(0);
+ });
+
+ it('erases the last added tag when backspacing on empty input', () => {
+ setupTagsInput(tagBlock);
+ addTag(plainInput, 'a');
+ addTag(plainInput, 'b');
+
+ const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true });
+ fancyText.dispatchEvent(ev);
+
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ });
+
+ it('adds new tag when comma is pressed', () => {
+ setupTagsInput(tagBlock);
+
+ const ev = new KeyboardEvent('keydown', { keyCode: 188, bubbles: true });
+ fancyText.value = 'a';
+ fancyText.dispatchEvent(ev);
+
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ expect(fancyText.value).toBe('');
+ });
+
+ it('adds new tag when enter is pressed', () => {
+ setupTagsInput(tagBlock);
+
+ const ev = new KeyboardEvent('keydown', { keyCode: 13, bubbles: true });
+ fancyText.value = 'a';
+ fancyText.dispatchEvent(ev);
+
+ expect($$('span.tag', fancyInput)).toHaveLength(1);
+ expect(fancyText.value).toBe('');
+ });
+});
diff --git a/assets/js/autocomplete.ts b/assets/js/autocomplete.ts
index 489392c3..61340b48 100644
--- a/assets/js/autocomplete.ts
+++ b/assets/js/autocomplete.ts
@@ -25,6 +25,14 @@ function restoreOriginalValue() {
if (isSearchField(inputField) && originalQuery) {
inputField.value = originalQuery;
+
+ if (selectedTerm) {
+ const [, selectedTermEnd] = selectedTerm[0];
+
+ inputField.setSelectionRange(selectedTermEnd, selectedTermEnd);
+ }
+
+ return;
}
if (originalTerm) {
diff --git a/assets/js/markdowntoolbar.ts b/assets/js/markdowntoolbar.ts
index f9ceb840..0b355e85 100644
--- a/assets/js/markdowntoolbar.ts
+++ b/assets/js/markdowntoolbar.ts
@@ -60,7 +60,7 @@ const markdownSyntax: Record = {
},
subscript: {
action: wrapSelection,
- options: { prefix: '%' },
+ options: { prefix: '~' },
},
quote: {
action: wrapLines,
diff --git a/assets/js/quick-tag.js b/assets/js/quick-tag.js
deleted file mode 100644
index 4457784a..00000000
--- a/assets/js/quick-tag.js
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Quick Tag
- */
-
-import store from './utils/store';
-import { $, $$, toggleEl, onLeftClick } from './utils/dom';
-import { fetchJson, handleError } from './utils/requests';
-
-const imageQueueStorage = 'quickTagQueue';
-const currentTagStorage = 'quickTagName';
-
-function currentQueue() {
- return store.get(imageQueueStorage) || [];
-}
-
-function currentTags() {
- return store.get(currentTagStorage) || '';
-}
-
-function getTagButton() {
- return $('.js-quick-tag');
-}
-
-function setTagButton(text) {
- $('.js-quick-tag--submit span').textContent = text;
-}
-
-function toggleActiveState() {
- toggleEl($('.js-quick-tag'), $('.js-quick-tag--abort'), $('.js-quick-tag--all'), $('.js-quick-tag--submit'));
-
- setTagButton(`Submit (${currentTags()})`);
-
- $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected'));
- $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected'));
- currentQueue().forEach(id =>
- $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')),
- );
-}
-
-function activate() {
- store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:'));
-
- if (currentTags()) toggleActiveState();
-}
-
-function reset() {
- store.remove(currentTagStorage);
- store.remove(imageQueueStorage);
-
- toggleActiveState();
-}
-
-function promptReset() {
- if (window.confirm('Are you sure you want to abort batch tagging?')) {
- reset();
- }
-}
-
-function submit() {
- setTagButton(`Wait... (${currentTags()})`);
-
- fetchJson('PUT', '/admin/batch/tags', {
- tags: currentTags(),
- image_ids: currentQueue(),
- })
- .then(handleError)
- .then(r => r.json())
- .then(data => {
- if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`);
-
- reset();
- });
-}
-
-function modifyImageQueue(mediaBox) {
- if (currentTags()) {
- const imageId = mediaBox.dataset.imageId;
- const queue = currentQueue();
- const isSelected = queue.includes(imageId);
-
- isSelected ? queue.splice(queue.indexOf(imageId), 1) : queue.push(imageId);
-
- $$(`.media-box__header[data-image-id="${imageId}"]`).forEach(el =>
- el.classList.toggle('media-box__header--selected'),
- );
-
- store.set(imageQueueStorage, queue);
- }
-}
-
-function toggleAllImages() {
- $$('#imagelist-container .media-box').forEach(modifyImageQueue);
-}
-
-function clickHandler(event) {
- const targets = {
- '.js-quick-tag': activate,
- '.js-quick-tag--abort': promptReset,
- '.js-quick-tag--submit': submit,
- '.js-quick-tag--all': toggleAllImages,
- '.media-box': modifyImageQueue,
- };
-
- for (const target in targets) {
- if (event.target && event.target.closest(target)) {
- targets[target](event.target.closest(target));
- currentTags() && event.preventDefault();
- }
- }
-}
-
-function setupQuickTag() {
- if (getTagButton() && currentTags()) toggleActiveState();
- if (getTagButton()) onLeftClick(clickHandler);
-}
-
-export { setupQuickTag };
diff --git a/assets/js/quick-tag.ts b/assets/js/quick-tag.ts
new file mode 100644
index 00000000..3d462cb2
--- /dev/null
+++ b/assets/js/quick-tag.ts
@@ -0,0 +1,124 @@
+/**
+ * Quick Tag
+ */
+
+import store from './utils/store';
+import { assertNotNull, assertNotUndefined } from './utils/assert';
+import { $, $$, toggleEl } from './utils/dom';
+import { fetchJson, handleError } from './utils/requests';
+import { delegate, leftClick } from './utils/events';
+
+const imageQueueStorage = 'quickTagQueue';
+const currentTagStorage = 'quickTagName';
+
+function currentQueue(): string[] {
+ return store.get(imageQueueStorage) || [];
+}
+
+function currentTags(): string {
+ return store.get(currentTagStorage) || '';
+}
+
+function setTagButton(text: string) {
+ assertNotNull($('.js-quick-tag--submit span')).textContent = text;
+}
+
+function toggleActiveState() {
+ toggleEl($$('.js-quick-tag,.js-quick-tag--abort,.js-quick-tag--all,.js-quick-tag--submit'));
+
+ setTagButton(`Submit (${currentTags()})`);
+
+ $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected'));
+ $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected'));
+
+ currentQueue().forEach(id =>
+ $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')),
+ );
+}
+
+function activate(event: Event) {
+ event.preventDefault();
+
+ store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:'));
+
+ if (currentTags()) {
+ toggleActiveState();
+ }
+}
+
+function reset() {
+ store.remove(currentTagStorage);
+ store.remove(imageQueueStorage);
+
+ toggleActiveState();
+}
+
+function promptReset(event: Event) {
+ event.preventDefault();
+
+ if (window.confirm('Are you sure you want to abort batch tagging?')) {
+ reset();
+ }
+}
+
+function submit(event: Event) {
+ event.preventDefault();
+
+ setTagButton(`Wait... (${currentTags()})`);
+
+ fetchJson('PUT', '/admin/batch/tags', {
+ tags: currentTags(),
+ image_ids: currentQueue(),
+ })
+ .then(handleError)
+ .then(r => r.json())
+ .then(data => {
+ if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`);
+
+ reset();
+ });
+}
+
+function modifyImageQueue(event: Event, mediaBox: HTMLDivElement) {
+ if (!currentTags()) {
+ return;
+ }
+
+ const imageId = assertNotUndefined(mediaBox.dataset.imageId);
+ const queue = currentQueue();
+ const isSelected = queue.includes(imageId);
+
+ if (isSelected) {
+ queue.splice(queue.indexOf(imageId), 1);
+ } else {
+ queue.push(imageId);
+ }
+
+ for (const boxHeader of $$(`.media-box__header[data-image-id="${imageId}"]`)) {
+ boxHeader.classList.toggle('media-box__header--selected');
+ }
+
+ store.set(imageQueueStorage, queue);
+ event.preventDefault();
+}
+
+function toggleAllImages(event: Event, _target: Element) {
+ for (const mediaBox of $$('#imagelist-container .media-box')) {
+ modifyImageQueue(event, mediaBox);
+ }
+}
+
+delegate(document, 'click', {
+ '.js-quick-tag': leftClick(activate),
+ '.js-quick-tag--abort': leftClick(promptReset),
+ '.js-quick-tag--submit': leftClick(submit),
+ '.js-quick-tag--all': leftClick(toggleAllImages),
+ '.media-box': leftClick(modifyImageQueue),
+});
+
+export function setupQuickTag() {
+ const tagButton = $('.js-quick-tag');
+ if (tagButton && currentTags()) {
+ toggleActiveState();
+ }
+}
diff --git a/assets/js/search.ts b/assets/js/search.ts
index 269497d2..eff8d98a 100644
--- a/assets/js/search.ts
+++ b/assets/js/search.ts
@@ -1,54 +1,85 @@
-import { $, $$ } from './utils/dom';
+import { assertNotNull, assertNotUndefined } from './utils/assert';
+import { $, $$, showEl, hideEl } from './utils/dom';
+import { delegate, leftClick } from './utils/events';
import { addTag } from './tagsinput';
-function showHelp(subject: string, type: string | null) {
- $$('[data-search-help]').forEach(helpBox => {
- if (helpBox.getAttribute('data-search-help') === type) {
- const searchSubject = $('.js-search-help-subject', helpBox);
-
- if (searchSubject) {
- searchSubject.textContent = subject;
- }
-
- helpBox.classList.remove('hidden');
- } else {
- helpBox.classList.add('hidden');
- }
- });
-}
-
-function prependToLast(field: HTMLInputElement, value: string) {
- const separatorIndex = field.value.lastIndexOf(',');
- const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1;
- field.value =
- field.value.slice(0, separatorIndex + advanceBy) + value + field.value.slice(separatorIndex + advanceBy);
-}
-
-function selectLast(field: HTMLInputElement, characterCount: number) {
+function focusAndSelectLast(field: HTMLInputElement, characterCount: number) {
field.focus();
-
field.selectionStart = field.value.length - characterCount;
field.selectionEnd = field.value.length;
}
-function executeFormHelper(e: PointerEvent) {
- if (!e.target) {
- return;
- }
+function prependToLast(field: HTMLInputElement, value: string) {
+ // Find the last comma in the input and advance past it
+ const separatorIndex = field.value.lastIndexOf(',');
+ const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1;
- const searchField = $('.js-search-field');
- const attr = (name: string) => e.target && (e.target as HTMLElement).getAttribute(name);
+ // Insert the value string at the new location
+ field.value = [
+ field.value.slice(0, separatorIndex + advanceBy),
+ value,
+ field.value.slice(separatorIndex + advanceBy),
+ ].join('');
+}
- if (attr('data-search-add')) addTag(searchField, attr('data-search-add'));
- if (attr('data-search-show-help')) showHelp((e.target as Node).textContent || '', attr('data-search-show-help'));
- if (attr('data-search-select-last') && searchField) {
- selectLast(searchField, parseInt(attr('data-search-select-last') || '', 10));
+function getAssociatedData(target: HTMLElement) {
+ const form = assertNotNull(target.closest('form'));
+ const input = assertNotNull($('.js-search-field', form));
+ const helpBoxes = $$('[data-search-help]', form);
+
+ return { input, helpBoxes };
+}
+
+function showHelp(helpBoxes: HTMLDivElement[], typeName: string, subject: string) {
+ for (const helpBox of helpBoxes) {
+ // Get the subject name span
+ const subjectName = assertNotNull($('.js-search-help-subject', helpBox));
+
+ // Take the appropriate action for this help box
+ if (helpBox.dataset.searchHelp === typeName) {
+ subjectName.textContent = subject;
+ showEl(helpBox);
+ } else {
+ hideEl(helpBox);
+ }
}
- if (attr('data-search-prepend') && searchField) prependToLast(searchField, attr('data-search-prepend') || '');
+}
+
+function onSearchAdd(_event: Event, target: HTMLAnchorElement) {
+ // Load form
+ const { input, helpBoxes } = getAssociatedData(target);
+
+ // Get data for this link
+ const addValue = assertNotUndefined(target.dataset.searchAdd);
+ const showHelpValue = assertNotUndefined(target.dataset.searchShowHelp);
+ const selectLastValue = target.dataset.searchSelectLast;
+
+ // Add the tag
+ addTag(input, addValue);
+
+ // Show associated help, if available
+ showHelp(helpBoxes, showHelpValue, assertNotNull(target.textContent));
+
+ // Select last characters, if requested
+ if (selectLastValue) {
+ focusAndSelectLast(input, Number(selectLastValue));
+ }
+}
+
+function onSearchPrepend(_event: Event, target: HTMLAnchorElement) {
+ // Load form
+ const { input } = getAssociatedData(target);
+
+ // Get data for this link
+ const prependValue = assertNotUndefined(target.dataset.searchPrepend);
+
+ // Prepend
+ prependToLast(input, prependValue);
}
export function setupSearch() {
- const form = $('.js-search-form');
-
- if (form) form.addEventListener('click', executeFormHelper as EventListener);
+ delegate(document, 'click', {
+ 'form.js-search-form a[data-search-add][data-search-show-help]': leftClick(onSearchAdd),
+ 'form.js-search-form a[data-search-prepend]': leftClick(onSearchPrepend),
+ });
}
diff --git a/assets/js/tagsinput.js b/assets/js/tagsinput.ts
similarity index 63%
rename from assets/js/tagsinput.js
rename to assets/js/tagsinput.ts
index bc05b3d7..74574906 100644
--- a/assets/js/tagsinput.js
+++ b/assets/js/tagsinput.ts
@@ -2,14 +2,20 @@
* Fancy tag editor.
*/
+import { assertNotNull, assertType } from './utils/assert';
import { $, $$, clearEl, removeEl, showEl, hideEl, escapeCss, escapeHtml } from './utils/dom';
+import { TermSuggestion } from './utils/suggestions';
-function setupTagsInput(tagBlock) {
- const [textarea, container] = $$('.js-taginput', tagBlock);
- const setup = $('.js-tag-block ~ button', tagBlock.parentNode);
- const inputField = $('input', container);
+export function setupTagsInput(tagBlock: HTMLDivElement) {
+ const form = assertNotNull(tagBlock.closest('form'));
+ const textarea = assertNotNull($('.js-taginput-plain', tagBlock));
+ const container = assertNotNull($('.js-taginput-fancy', tagBlock));
+ const parentField = assertNotNull(tagBlock.parentElement);
+ const setup = assertNotNull($('.js-tag-block ~ button', parentField));
+ const inputField = assertNotNull($('input', container));
+ const submitButton = assertNotNull($('[type="submit"]', form));
- let tags = [];
+ let tags: string[] = [];
// Load in the current tag set from the textarea
setup.addEventListener('click', importTags);
@@ -27,7 +33,7 @@ function setupTagsInput(tagBlock) {
inputField.addEventListener('keydown', handleKeyEvent);
// Respond to autocomplete form clicks
- inputField.addEventListener('autocomplete', handleAutocomplete);
+ inputField.addEventListener('autocomplete', handleAutocomplete as EventListener);
// Respond to Ctrl+Enter shortcut
tagBlock.addEventListener('keydown', handleCtrlEnter);
@@ -35,19 +41,19 @@ function setupTagsInput(tagBlock) {
// TODO: Cleanup this bug fix
// Switch to fancy tagging if user settings want it
if (fancyEditorRequested(tagBlock)) {
- showEl($$('.js-taginput-fancy'));
- showEl($$('.js-taginput-hide'));
- hideEl($$('.js-taginput-plain'));
- hideEl($$('.js-taginput-show'));
+ showEl($$('.js-taginput-fancy'));
+ showEl($$('.js-taginput-hide'));
+ hideEl($$('.js-taginput-plain'));
+ hideEl($$('.js-taginput-show'));
importTags();
}
- function handleAutocomplete(event) {
+ function handleAutocomplete(event: CustomEvent) {
insertTag(event.detail.value);
inputField.focus();
}
- function handleAddTag(event) {
+ function handleAddTag(event: AddtagEvent) {
// Ignore if not in tag edit mode
if (container.classList.contains('hidden')) return;
@@ -55,14 +61,16 @@ function setupTagsInput(tagBlock) {
event.stopPropagation();
}
- function handleTagClear(event) {
- if (event.target.dataset.tagName) {
+ function handleTagClear(event: Event) {
+ const target = assertType(event.target, HTMLElement);
+
+ if (target.dataset.tagName) {
event.preventDefault();
- removeTag(event.target.dataset.tagName, event.target.parentNode);
+ removeTag(target.dataset.tagName, assertNotNull(target.parentElement));
}
}
- function handleKeyEvent(event) {
+ function handleKeyEvent(event: KeyboardEvent) {
const { keyCode, ctrlKey, shiftKey } = event;
// allow form submission with ctrl+enter if no text was typed
@@ -73,7 +81,7 @@ function setupTagsInput(tagBlock) {
// backspace on a blank input field
if (keyCode === 8 && inputField.value === '') {
event.preventDefault();
- const erased = $('.tag:last-of-type', container);
+ const erased = $('.tag:last-of-type', container);
if (erased) removeTag(tags[tags.length - 1], erased);
}
@@ -86,14 +94,14 @@ function setupTagsInput(tagBlock) {
}
}
- function handleCtrlEnter(event) {
+ function handleCtrlEnter(event: KeyboardEvent) {
const { keyCode, ctrlKey } = event;
if (keyCode !== 13 || !ctrlKey) return;
- $('[type="submit"]', tagBlock.closest('form')).click();
+ submitButton.click();
}
- function insertTag(name) {
+ function insertTag(name: string) {
name = name.trim(); // eslint-disable-line no-param-reassign
// Add if not degenerate or already present
@@ -102,9 +110,9 @@ function setupTagsInput(tagBlock) {
// Remove instead if the tag name starts with a minus
if (name[0] === '-') {
name = name.slice(1); // eslint-disable-line no-param-reassign
- const tagLink = $(`[data-tag-name="${escapeCss(name)}"]`, container);
+ const tagLink = assertNotNull($(`[data-tag-name="${escapeCss(name)}"]`, container));
- return removeTag(name, tagLink.parentNode);
+ return removeTag(name, assertNotNull(tagLink.parentElement));
}
tags.push(name);
@@ -116,7 +124,7 @@ function setupTagsInput(tagBlock) {
inputField.value = '';
}
- function removeTag(name, element) {
+ function removeTag(name: string, element: HTMLElement) {
removeEl(element);
// Remove the tag from the list
@@ -134,7 +142,7 @@ function setupTagsInput(tagBlock) {
}
}
-function fancyEditorRequested(tagBlock) {
+function fancyEditorRequested(tagBlock: HTMLDivElement) {
// Check whether the user made the fancy editor the default for each type of tag block.
return (
(window.booru.fancyTagUpload && tagBlock.classList.contains('fancy-tag-upload')) ||
@@ -142,19 +150,17 @@ function fancyEditorRequested(tagBlock) {
);
}
-function setupTagListener() {
+export function setupTagListener() {
document.addEventListener('addtag', event => {
if (event.target.value) event.target.value += ', ';
event.target.value += event.detail.name;
});
}
-function addTag(textarea, name) {
+export function addTag(textarea: HTMLInputElement | HTMLTextAreaElement, name: string) {
textarea.dispatchEvent(new CustomEvent('addtag', { detail: { name }, bubbles: true }));
}
-function reloadTagsInput(textarea) {
+export function reloadTagsInput(textarea: HTMLInputElement | HTMLTextAreaElement) {
textarea.dispatchEvent(new CustomEvent('reload'));
}
-
-export { setupTagsInput, setupTagListener, addTag, reloadTagsInput };
diff --git a/assets/package-lock.json b/assets/package-lock.json
index 0f73ac68..3024e2ec 100644
--- a/assets/package-lock.json
+++ b/assets/package-lock.json
@@ -2380,9 +2380,9 @@
}
},
"node_modules/cross-spawn": {
- "version": "7.0.3",
- "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
- "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "version": "7.0.6",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+ "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
diff --git a/assets/test/vitest-setup.ts b/assets/test/vitest-setup.ts
index 27688447..d533bd4d 100644
--- a/assets/test/vitest-setup.ts
+++ b/assets/test/vitest-setup.ts
@@ -7,6 +7,8 @@ import { fireEvent } from '@testing-library/dom';
window.booru = {
timeAgo: () => {},
csrfToken: 'mockCsrfToken',
+ fancyTagEdit: true,
+ fancyTagUpload: true,
hiddenTag: '/mock-tagblocked.svg',
hiddenTagList: [],
hideStaffTools: 'true',
diff --git a/assets/types/booru-object.d.ts b/assets/types/booru-object.d.ts
index 4154385c..8b04cced 100644
--- a/assets/types/booru-object.d.ts
+++ b/assets/types/booru-object.d.ts
@@ -73,6 +73,14 @@ interface BooruObject {
* List of image IDs in the current gallery.
*/
galleryImages?: number[];
+ /**
+ * Fancy tag setting for uploading images.
+ */
+ fancyTagUpload: boolean;
+ /**
+ * Fancy tag setting for editing images.
+ */
+ fancyTagEdit: boolean;
}
declare global {
diff --git a/assets/types/tags.ts b/assets/types/tags.ts
new file mode 100644
index 00000000..b132dbf0
--- /dev/null
+++ b/assets/types/tags.ts
@@ -0,0 +1,20 @@
+export {};
+
+declare global {
+ interface Addtag {
+ name: string;
+ }
+
+ interface AddtagEvent extends CustomEvent {
+ target: HTMLInputElement | HTMLTextAreaElement;
+ }
+
+ interface ReloadEvent extends CustomEvent {
+ target: HTMLInputElement | HTMLTextAreaElement;
+ }
+
+ interface GlobalEventHandlersEventMap {
+ addtag: AddtagEvent;
+ reload: ReloadEvent;
+ }
+}
diff --git a/docker-compose.yml b/docker-compose.yml
index 5b102408..a1233418 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -64,16 +64,14 @@ services:
- POSTGRES_PASSWORD=postgres
volumes:
- postgres_data:/var/lib/postgresql/data
- logging:
- driver: "none"
+ attach: false
opensearch:
image: opensearchproject/opensearch:2.16.0
volumes:
- opensearch_data:/usr/share/opensearch/data
- ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
- logging:
- driver: "none"
+ attach: false
ulimits:
nofile:
soft: 65536
@@ -81,8 +79,7 @@ services:
valkey:
image: valkey/valkey:8.0-alpine
- logging:
- driver: "none"
+ attach: false
files:
image: andrewgaul/s3proxy:sha-4976e17
@@ -90,6 +87,7 @@ services:
- JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3
volumes:
- .:/srv/philomena
+ attach: false
web:
build:
@@ -106,8 +104,7 @@ services:
environment:
- AWS_ACCESS_KEY_ID=local-identity
- AWS_SECRET_ACCESS_KEY=local-credential
- logging:
- driver: "none"
+ attach: false
depends_on:
- app
ports:
diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 87ab48d3..61c61f87 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -121,7 +121,9 @@ defmodule Mix.Tasks.UploadToS3 do
end
defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
- Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+ queryable
+ |> Batch.record_batches(batch_size: batch_size)
+ |> Enum.each(fn models ->
models
|> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name),
timeout: :infinity
@@ -142,7 +144,9 @@ defmodule Mix.Tasks.UploadToS3 do
end
defp upload_images(queryable, batch_size, file_root, new_file_root) do
- Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+ queryable
+ |> Batch.record_batches(batch_size: batch_size)
+ |> Enum.each(fn models ->
models
|> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity)
|> Stream.run()
diff --git a/lib/philomena/data_exports/aggregator.ex b/lib/philomena/data_exports/aggregator.ex
new file mode 100644
index 00000000..5dcb82b3
--- /dev/null
+++ b/lib/philomena/data_exports/aggregator.ex
@@ -0,0 +1,171 @@
+defmodule Philomena.DataExports.Aggregator do
+ @moduledoc """
+ Data generation module for data export logic.
+ """
+
+ import Ecto.Query
+ alias PhilomenaQuery.Batch
+
+ # Direct PII
+ alias Philomena.Donations.Donation
+ alias Philomena.UserFingerprints.UserFingerprint
+ alias Philomena.UserIps.UserIp
+ alias Philomena.UserNameChanges.UserNameChange
+ alias Philomena.Users.User
+
+ # UGC for export
+ alias Philomena.ArtistLinks.ArtistLink
+ alias Philomena.Badges.Award
+ alias Philomena.Comments.Comment
+ alias Philomena.Commissions.Commission
+ alias Philomena.DnpEntries.DnpEntry
+ alias Philomena.DuplicateReports.DuplicateReport
+ alias Philomena.Filters.Filter
+ alias Philomena.ImageFaves.ImageFave
+ alias Philomena.ImageHides.ImageHide
+ alias Philomena.ImageVotes.ImageVote
+ alias Philomena.Images.Image
+ alias Philomena.PollVotes.PollVote
+ alias Philomena.Posts.Post
+ alias Philomena.Reports.Report
+ alias Philomena.SourceChanges.SourceChange
+ alias Philomena.TagChanges.TagChange
+ alias Philomena.Topics.Topic
+ alias Philomena.Bans.User, as: UserBan
+
+ # Direct UGC from form submission
+ @user_columns [
+ :created_at,
+ :name,
+ :email,
+ :description,
+ :current_filter_id,
+ :spoiler_type,
+ :theme,
+ :images_per_page,
+ :show_large_thumbnails,
+ :show_sidebar_and_watched_images,
+ :fancy_tag_field_on_upload,
+ :fancy_tag_field_on_edit,
+ :fancy_tag_field_in_settings,
+ :autorefresh_by_default,
+ :anonymous_by_default,
+ :comments_newest_first,
+ :comments_always_jump_to_last,
+ :comments_per_page,
+ :watch_on_reply,
+ :watch_on_new_topic,
+ :watch_on_upload,
+ :messages_newest_first,
+ :serve_webm,
+ :no_spoilered_in_watched,
+ :watched_images_query_str,
+ :watched_images_exclude_str,
+ :use_centered_layout,
+ :personal_title,
+ :hide_vote_counts,
+ :scale_large_images
+ ]
+
+ # All these also have created_at and are selected by user_id
+ @indirect_columns [
+ {Donation, [:email, :amount, :fee, :note]},
+ {UserFingerprint, [:fingerprint, :uses, :updated_at]},
+ {UserIp, [:ip, :uses, :updated_at]},
+ {UserNameChange, [:name]},
+ {ArtistLink, [:aasm_state, :uri, :public, :tag_id]},
+ {Award, [:label, :badge_name, :badge_id]},
+ {Comment,
+ [
+ :ip,
+ :fingerprint,
+ :user_agent,
+ :referrer,
+ :anonymous,
+ :image_id,
+ :edited_at,
+ :edit_reason,
+ :body
+ ]},
+ {Commission,
+ [:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]},
+ {DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions],
+ :requesting_user_id},
+ {DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]},
+ {Filter,
+ [
+ :name,
+ :description,
+ :public,
+ :hidden_complex_str,
+ :spoilered_complex_str,
+ :hidden_tag_ids,
+ :spoilered_tag_ids
+ ]},
+ {ImageFave, [:image_id], :user_id, :image_id},
+ {ImageHide, [:image_id], :user_id, :image_id},
+ {ImageVote, [:image_id, :up], :user_id, :image_id},
+ {Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]},
+ {PollVote, [:rank, :poll_option_id]},
+ {Post,
+ [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]},
+ {Report,
+ [:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]},
+ {SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]},
+ {TagChange,
+ [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]},
+ {Topic, [:title, :anonymous, :forum_id]},
+ {UserBan, [:reason, :generated_ban_id]}
+ ]
+
+ @doc """
+ Get all of the export data for the given user.
+ """
+ def get_for_user(user_id) do
+ [select_user(user_id)] ++ select_indirect(user_id)
+ end
+
+ defp select_user(user_id) do
+ select_schema_by_key(user_id, User, @user_columns, :id)
+ end
+
+ defp select_indirect(user_id) do
+ Enum.map(@indirect_columns, fn
+ {schema_name, columns} ->
+ select_schema_by_key(user_id, schema_name, columns)
+
+ {schema_name, columns, key_column} ->
+ select_schema_by_key(user_id, schema_name, columns, key_column)
+
+ {schema_name, columns, key_column, id_field} ->
+ select_schema_by_key(user_id, schema_name, columns, key_column, id_field)
+ end)
+ end
+
+ defp select_schema_by_key(
+ user_id,
+ schema_name,
+ columns,
+ key_column \\ :user_id,
+ id_field \\ :id
+ ) do
+ table_name = schema_name.__schema__(:source)
+ columns = [:created_at] ++ columns
+
+ {"#{table_name}.jsonl",
+ schema_name
+ |> where([s], field(s, ^key_column) == ^user_id)
+ |> select([s], map(s, ^columns))
+ |> Batch.records(id_field: id_field)
+ |> results_as_json_lines()}
+ end
+
+ defp results_as_json_lines(list_of_maps) do
+ Stream.map(list_of_maps, fn map ->
+ map
+ |> Map.new(fn {k, v} -> {k, to_string(v)} end)
+ |> Jason.encode!()
+ |> Kernel.<>("\n")
+ end)
+ end
+end
diff --git a/lib/philomena/data_exports/zip_generator.ex b/lib/philomena/data_exports/zip_generator.ex
new file mode 100644
index 00000000..8c5aaf7e
--- /dev/null
+++ b/lib/philomena/data_exports/zip_generator.ex
@@ -0,0 +1,56 @@
+defmodule Philomena.DataExports.ZipGenerator do
+ @moduledoc """
+ ZIP file generator for an export.
+ """
+
+ alias Philomena.Native
+
+ @doc """
+ Write the ZIP file for the given aggregate data.
+
+ Expects a list of 2-tuples, with the first element being the name of the
+ file to generate, and the second element being a stream which generates the
+ binary contents of the file.
+ """
+ @spec generate(Path.t(), Enumerable.t()) :: :ok | atom()
+ def generate(filename, aggregate) do
+ case Native.zip_open_writer(filename) do
+ {:ok, zip} ->
+ stream_aggregate(zip, aggregate)
+
+ error ->
+ error
+ end
+ end
+
+ @spec stream_aggregate(reference(), Enumerable.t()) :: {:ok, reference()} | :error
+ defp stream_aggregate(zip, aggregate) do
+ aggregate
+ |> Enum.reduce_while(:ok, fn {name, content_stream}, _ ->
+ with :ok <- Native.zip_start_file(zip, name),
+ :ok <- stream_file_data(zip, content_stream) do
+ {:cont, :ok}
+ else
+ error ->
+ {:halt, error}
+ end
+ end)
+ |> case do
+ :ok ->
+ Native.zip_finish(zip)
+
+ error ->
+ error
+ end
+ end
+
+ @spec stream_file_data(reference(), Enumerable.t(iodata())) :: :ok | :error
+ defp stream_file_data(zip, content_stream) do
+ Enum.reduce_while(content_stream, :ok, fn iodata, _ ->
+ case Native.zip_write(zip, IO.iodata_to_binary(iodata)) do
+ :ok -> {:cont, :ok}
+ error -> {:halt, error}
+ end
+ end)
+ end
+end
diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex
index a67dc33f..14eeaa17 100644
--- a/lib/philomena/native.ex
+++ b/lib/philomena/native.ex
@@ -11,4 +11,16 @@ defmodule Philomena.Native do
@spec camo_image_url(String.t()) :: String.t()
def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
+
+ @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()}
+ def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded)
+
+ @spec zip_start_file(reference(), String.t()) :: :ok | :error
+ def zip_start_file(_zip, _name), do: :erlang.nif_error(:nif_not_loaded)
+
+ @spec zip_write(reference(), binary()) :: :ok | :error
+ def zip_write(_zip, _data), do: :erlang.nif_error(:nif_not_loaded)
+
+ @spec zip_finish(reference()) :: :ok | :error
+ def zip_finish(_zip), do: :erlang.nif_error(:nif_not_loaded)
end
diff --git a/lib/philomena/user_downvote_wipe.ex b/lib/philomena/user_downvote_wipe.ex
index 6fb217fe..b958ab9b 100644
--- a/lib/philomena/user_downvote_wipe.ex
+++ b/lib/philomena/user_downvote_wipe.ex
@@ -15,7 +15,8 @@ defmodule Philomena.UserDownvoteWipe do
ImageVote
|> where(user_id: ^user.id, up: false)
- |> Batch.query_batches([id_field: :image_id], fn queryable ->
+ |> Batch.query_batches(id_field: :image_id)
+ |> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id))
{count, nil} =
@@ -31,7 +32,8 @@ defmodule Philomena.UserDownvoteWipe do
if upvotes_and_faves_too do
ImageVote
|> where(user_id: ^user.id, up: true)
- |> Batch.query_batches([id_field: :image_id], fn queryable ->
+ |> Batch.query_batches(id_field: :image_id)
+ |> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id))
{count, nil} =
@@ -46,7 +48,8 @@ defmodule Philomena.UserDownvoteWipe do
ImageFave
|> where(user_id: ^user.id)
- |> Batch.query_batches([id_field: :image_id], fn queryable ->
+ |> Batch.query_batches(id_field: :image_id)
+ |> Enum.each(fn queryable ->
{_, image_ids} = Repo.delete_all(select(queryable, [i_f], i_f.image_id))
{count, nil} =
diff --git a/lib/philomena/workers/tag_change_revert_worker.ex b/lib/philomena/workers/tag_change_revert_worker.ex
index 519b8404..80058a31 100644
--- a/lib/philomena/workers/tag_change_revert_worker.ex
+++ b/lib/philomena/workers/tag_change_revert_worker.ex
@@ -27,7 +27,9 @@ defmodule Philomena.TagChangeRevertWorker do
batch_size = attributes["batch_size"] || 100
attributes = Map.delete(attributes, "batch_size")
- Batch.query_batches(queryable, [batch_size: batch_size], fn queryable ->
+ queryable
+ |> Batch.query_batches(batch_size: batch_size)
+ |> Enum.each(fn queryable ->
ids = Repo.all(select(queryable, [tc], tc.id))
TagChanges.mass_revert(ids, cast_ip(atomify_keys(attributes)))
end)
diff --git a/lib/philomena_proxy/scrapers.ex b/lib/philomena_proxy/scrapers.ex
index a96f0817..08674d44 100644
--- a/lib/philomena_proxy/scrapers.ex
+++ b/lib/philomena_proxy/scrapers.ex
@@ -21,6 +21,7 @@ defmodule PhilomenaProxy.Scrapers do
}
@scrapers [
+ PhilomenaProxy.Scrapers.Bluesky,
PhilomenaProxy.Scrapers.Deviantart,
PhilomenaProxy.Scrapers.Pillowfort,
PhilomenaProxy.Scrapers.Twitter,
diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex
new file mode 100644
index 00000000..e67672b2
--- /dev/null
+++ b/lib/philomena_proxy/scrapers/bluesky.ex
@@ -0,0 +1,76 @@
+defmodule PhilomenaProxy.Scrapers.Bluesky do
+ @moduledoc false
+
+ alias PhilomenaProxy.Scrapers.Scraper
+ alias PhilomenaProxy.Scrapers
+
+ @behaviour Scraper
+
+ @url_regex ~r|https://bsky\.app/profile/([^/]+)/post/([^/?#]+)|
+ @fullsize_image_regex ~r|.*/img/feed_fullsize/plain/([^/]+)/([^@]+).*|
+ @blob_image_url_pattern "https://bsky.social/xrpc/com.atproto.sync.getBlob/?did=\\1&cid=\\2"
+
+ @spec can_handle?(URI.t(), String.t()) :: boolean()
+ def can_handle?(_uri, url) do
+ String.match?(url, @url_regex)
+ end
+
+ @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result()
+ def scrape(_uri, url) do
+ [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first)
+
+ did = fetch_did(handle)
+
+ api_url_get_posts =
+ "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}"
+
+ post_json =
+ api_url_get_posts
+ |> PhilomenaProxy.Http.get()
+ |> json!()
+ |> Map.fetch!("posts")
+ |> hd()
+
+ %{
+ source_url: url,
+ author_name: domain_first_component(post_json["author"]["handle"]),
+ description: post_json["record"]["text"],
+ images:
+ Enum.map(
+ post_json["embed"]["images"],
+ &%{
+ url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern),
+ camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"])
+ }
+ )
+ }
+ end
+
+ defp fetch_did(handle) do
+ case handle do
+ <<"did:", _rest::binary>> ->
+ handle
+
+ _ ->
+ api_url_resolve_handle =
+ "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}"
+
+ api_url_resolve_handle
+ |> PhilomenaProxy.Http.get()
+ |> json!()
+ |> Map.fetch!("did")
+ end
+ end
+
+ defp domain_first_component(domain) do
+ case String.split(domain, ".") do
+ [name | _] ->
+ name
+
+ _ ->
+ domain
+ end
+ end
+
+ defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body)
+end
diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex
index 918a3b5e..bb9c2c74 100644
--- a/lib/philomena_query/batch.ex
+++ b/lib/philomena_query/batch.ex
@@ -25,24 +25,32 @@ defmodule PhilomenaQuery.Batch do
@type id_field :: {:id_field, atom()}
@type batch_options :: [batch_size() | id_field()]
- @typedoc """
- The callback for `record_batches/3`.
+ @doc """
+ Stream schema structures on a queryable, using batches to avoid locking.
+
+ Valid options:
+ * `batch_size` (integer) - the number of records to load per batch
+ * `id_field` (atom) - the name of the field containing the ID
+
+ ## Example
+
+ queryable = from i in Image, where: i.image_width >= 1920
+
+ queryable
+ |> PhilomenaQuery.Batch.record_batches()
+ |> Enum.each(fn image -> IO.inspect(image.id) end)
- Takes a list of schema structs which were returned in the batch. Return value is ignored.
"""
- @type record_batch_callback :: ([struct()] -> any())
-
- @typedoc """
- The callback for `query_batches/3`.
-
- Takes an `m:Ecto.Query` that can be processed with `m:Philomena.Repo` query commands, such
- as `Philomena.Repo.update_all/3` or `Philomena.Repo.delete_all/2`. Return value is ignored.
- """
- @type query_batch_callback :: ([Ecto.Query.t()] -> any())
+ @spec records(queryable(), batch_options()) :: Enumerable.t()
+ def records(queryable, opts \\ []) do
+ queryable
+ |> query_batches(opts)
+ |> Stream.flat_map(&Repo.all/1)
+ end
@doc """
- Execute a callback with lists of schema structures on a queryable,
- using batches to avoid locking.
+ Stream lists of schema structures on a queryable, using batches to avoid
+ locking.
Valid options:
* `batch_size` (integer) - the number of records to load per batch
@@ -56,16 +64,20 @@ defmodule PhilomenaQuery.Batch do
Enum.each(images, &IO.inspect(&1.id))
end
- PhilomenaQuery.Batch.record_batches(queryable, cb)
+ queryable
+ |> PhilomenaQuery.Batch.record_batches()
+ |> Enum.each(cb)
"""
- @spec record_batches(queryable(), batch_options(), record_batch_callback()) :: []
- def record_batches(queryable, opts \\ [], callback) do
- query_batches(queryable, opts, &callback.(Repo.all(&1)))
+ @spec record_batches(queryable(), batch_options()) :: Enumerable.t()
+ def record_batches(queryable, opts \\ []) do
+ queryable
+ |> query_batches(opts)
+ |> Stream.map(&Repo.all/1)
end
@doc """
- Execute a callback with bulk queries on a queryable, using batches to avoid locking.
+ Stream bulk queries on a queryable, using batches to avoid locking.
Valid options:
* `batch_size` (integer) - the number of records to load per batch
@@ -76,41 +88,36 @@ defmodule PhilomenaQuery.Batch do
> If you are looking to receive schema structures (e.g., you are querying for `Image`s,
> and you want to receive `Image` objects, then use `record_batches/3` instead.
- An `m:Ecto.Query` which selects all IDs in the current batch is passed into the callback
- during each invocation.
+ `m:Ecto.Query` structs which select the IDs in each batch are streamed out.
## Example
queryable = from ui in ImageVote, where: ui.user_id == 1234
- opts = [id_field: :image_id]
-
- cb = fn bulk_query ->
- Repo.delete_all(bulk_query)
- end
-
- PhilomenaQuery.Batch.query_batches(queryable, opts, cb)
+ queryable
+ |> PhilomenaQuery.Batch.query_batches(id_field: :image_id)
+ |> Enum.each(fn batch_query -> Repo.delete_all(batch_query) end)
"""
- @spec query_batches(queryable(), batch_options(), query_batch_callback()) :: []
- def query_batches(queryable, opts \\ [], callback) do
- ids = load_ids(queryable, -1, opts)
-
- query_batches(queryable, opts, callback, ids)
- end
-
- defp query_batches(_queryable, _opts, _callback, []), do: []
-
- defp query_batches(queryable, opts, callback, ids) do
+ @spec query_batches(queryable(), batch_options()) :: Enumerable.t(Ecto.Query.t())
+ def query_batches(queryable, opts \\ []) do
id_field = Keyword.get(opts, :id_field, :id)
- queryable
- |> where([m], field(m, ^id_field) in ^ids)
- |> callback.()
+ Stream.unfold(
+ load_ids(queryable, -1, opts),
+ fn
+ [] ->
+ # Stop when no more results are produced
+ nil
- ids = load_ids(queryable, Enum.max(ids), opts)
+ ids ->
+ # Process results and output next query
+ output = where(queryable, [m], field(m, ^id_field) in ^ids)
+ next_ids = load_ids(queryable, Enum.max(ids), opts)
- query_batches(queryable, opts, callback, ids)
+ {output, next_ids}
+ end
+ )
end
defp load_ids(queryable, max_id, opts) do
@@ -118,8 +125,9 @@ defmodule PhilomenaQuery.Batch do
batch_size = Keyword.get(opts, :batch_size, 1000)
queryable
- |> exclude(:preload)
|> exclude(:order_by)
+ |> exclude(:preload)
+ |> exclude(:select)
|> order_by(asc: ^id_field)
|> where([m], field(m, ^id_field) > ^max_id)
|> select([m], field(m, ^id_field))
diff --git a/lib/philomena_query/search.ex b/lib/philomena_query/search.ex
index 2519e580..cd02137c 100644
--- a/lib/philomena_query/search.ex
+++ b/lib/philomena_query/search.ex
@@ -199,11 +199,13 @@ defmodule PhilomenaQuery.Search do
Search.reindex(query, Image, batch_size: 5000)
"""
- @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: []
+ @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok
def reindex(queryable, module, opts \\ []) do
index = @policy.index_for(module)
- Batch.record_batches(queryable, opts, fn records ->
+ queryable
+ |> Batch.record_batches(opts)
+ |> Enum.each(fn records ->
lines =
Enum.flat_map(records, fn record ->
doc = index.as_json(record)
diff --git a/lib/philomena_web/controllers/image/source_controller.ex b/lib/philomena_web/controllers/image/source_controller.ex
index 373dcc10..2dbc4f47 100644
--- a/lib/philomena_web/controllers/image/source_controller.ex
+++ b/lib/philomena_web/controllers/image/source_controller.ex
@@ -4,6 +4,7 @@ defmodule PhilomenaWeb.Image.SourceController do
alias Philomena.SourceChanges.SourceChange
alias Philomena.UserStatistics
alias Philomena.Images.Image
+ alias Philomena.Images.Source
alias Philomena.Images
alias Philomena.Repo
import Ecto.Query
@@ -41,7 +42,9 @@ defmodule PhilomenaWeb.Image.SourceController do
PhilomenaWeb.Api.Json.ImageView.render("show.json", %{image: image, interactions: []})
)
- changeset = Images.change_image(image)
+ changeset =
+ %{image | sources: sources_for_edit(image.sources)}
+ |> Images.change_image()
source_change_count =
SourceChange
@@ -74,4 +77,9 @@ defmodule PhilomenaWeb.Image.SourceController do
)
end
end
+
+ # TODO: this is duplicated in ImageController
+ defp sources_for_edit(), do: [%Source{}]
+ defp sources_for_edit([]), do: sources_for_edit()
+ defp sources_for_edit(sources), do: sources
end
diff --git a/lib/philomena_web/controllers/image_controller.ex b/lib/philomena_web/controllers/image_controller.ex
index c66c2fe7..2eeb3c5a 100644
--- a/lib/philomena_web/controllers/image_controller.ex
+++ b/lib/philomena_web/controllers/image_controller.ex
@@ -219,6 +219,7 @@ defmodule PhilomenaWeb.ImageController do
end
end
+ # TODO: this is duplicated in Image.SourceController
defp sources_for_edit(), do: [%Source{}]
defp sources_for_edit([]), do: sources_for_edit()
defp sources_for_edit(sources), do: sources
diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex
index 7caff5c9..0b108008 100644
--- a/lib/philomena_web/markdown_renderer.ex
+++ b/lib/philomena_web/markdown_renderer.ex
@@ -75,8 +75,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do
defp render_representations(images, conn) do
loaded_images = load_images(images)
- images
- |> Enum.map(fn group ->
+ Map.new(images, fn group ->
img = loaded_images[Enum.at(group, 0)]
text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}"
@@ -131,8 +130,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do
|> Phoenix.HTML.Safe.to_iodata()
|> IO.iodata_to_binary()
- [text, string_contents]
+ {text, string_contents}
end)
- |> Map.new(fn [id, html] -> {id, html} end)
end
end
diff --git a/lib/philomena_web/templates/commission/_directory_results.html.slime b/lib/philomena_web/templates/commission/_directory_results.html.slime
index 8071b52f..47883dc7 100644
--- a/lib/philomena_web/templates/commission/_directory_results.html.slime
+++ b/lib/philomena_web/templates/commission/_directory_results.html.slime
@@ -62,3 +62,6 @@ elixir:
- true ->
p We couldn't find any commission listings to display. Sorry!
+
+ .block__header.page__header
+ .page__pagination = pagination
diff --git a/lib/philomena_web/templates/search/_form.html.slime b/lib/philomena_web/templates/search/_form.html.slime
index d1ceb34d..8ffeef48 100644
--- a/lib/philomena_web/templates/search/_form.html.slime
+++ b/lib/philomena_web/templates/search/_form.html.slime
@@ -1,7 +1,7 @@
h1 Search
= form_for :search, ~p"/search", [id: "searchform", method: "get", class: "js-search-form", enforce_utf8: false], fn f ->
- = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"]
+ = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"], data: [ac: "true", ac_min_length: 3, ac_mode: "search"]
.block
.block__header.flex
diff --git a/mix.exs b/mix.exs
index 4032d859..1dde7ea0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -77,7 +77,7 @@ defmodule Philomena.MixProject do
{:mail, "~> 0.3.0"},
# Markdown
- {:rustler, "~> 0.27"},
+ {:rustler, "~> 0.35"},
# Linting
{:credo, "~> 1.6", only: [:dev, :test], runtime: false},
diff --git a/mix.lock b/mix.lock
index 7f179ae0..35d487fe 100644
--- a/mix.lock
+++ b/mix.lock
@@ -5,14 +5,14 @@
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"canada": {:hex, :canada, "1.0.2", "040e4c47609b0a67d5773ac1fbe5e99f840cef173d69b739beda7c98453e0770", [:mix], [], "hexpm", "4269f74153fe89583fe50bd4d5de57bfe01f31258a6b676d296f3681f1483c68"},
"canary": {:git, "https://github.com/marcinkoziej/canary.git", "704debde7a2c0600f78c687807884bf37c45bd79", [ref: "704debde7a2c0600f78c687807884bf37c45bd79"]},
- "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"},
+ "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"},
"credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"},
"credo_envvar": {:hex, :credo_envvar, "0.1.4", "40817c10334e400f031012c0510bfa0d8725c19d867e4ae39cf14f2cbebc3b20", [:mix], [{:credo, "~> 1.0", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "5055cdb4bcbaf7d423bc2bb3ac62b4e2d825e2b1e816884c468dee59d0363009"},
"credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
"db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
- "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
+ "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"},
"dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
"earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"},
"ecto": {:hex, :ecto, "3.12.1", "626765f7066589de6fa09e0876a253ff60c3d00870dd3a1cd696e2ba67bfceea", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df0045ab9d87be947228e05a8d153f3e06e0d05ab10c3b3cc557d2f7243d1940"},
@@ -27,7 +27,7 @@
"expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"},
"exq": {:hex, :exq, "0.19.0", "06eb92944dad39f0954dc8f63190d3e24d11734eef88cf5800883e57ebf74f3c", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 6.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "24fc0ebdd87cc7406e1034fb46c2419f9c8a362f0ec634d23b6b819514d36390"},
"file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
- "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
+ "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"},
"gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"},
"hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
"inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"},
@@ -62,8 +62,8 @@
"qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"},
"redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"},
"remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"},
- "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"},
- "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"},
+ "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
+ "rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"},
"scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"},
"scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]},
"secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"},
@@ -71,7 +71,7 @@
"sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"},
"sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
"swoosh": {:hex, :swoosh, "1.16.10", "04be6e2eb1a31aa0aa21a731175c81cc3998189456a92daf13d44a5c754afcf5", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "756be04db173c0cbe318f1dfe2bcc88aa63aed78cf5a4b02b61b36ee11fc716a"},
- "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+ "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
"thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"},
"toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock
index 0ca74719..54aa5826 100644
--- a/native/philomena/Cargo.lock
+++ b/native/philomena/Cargo.lock
@@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 3
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
[[package]]
name = "aho-corasick"
version = "1.1.3"
@@ -11,12 +17,46 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "arbitrary"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
+dependencies = [
+ "derive_arbitrary",
+]
+
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+[[package]]
+name = "bon"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509"
+dependencies = [
+ "bon-macros",
+ "rustversion",
+]
+
+[[package]]
+name = "bon-macros"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33"
+dependencies = [
+ "darling",
+ "ident_case",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
[[package]]
name = "bumpalo"
version = "3.16.0"
@@ -29,6 +69,16 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+[[package]]
+name = "caseless"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
+dependencies = [
+ "regex",
+ "unicode-normalization",
+]
+
[[package]]
name = "cc"
version = "1.0.94"
@@ -43,12 +93,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "comrak"
-version = "0.24.1"
-source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a"
+version = "0.29.0"
+source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.2#00ac2a12d5797feb0ceba9a98487451ab65593fe"
dependencies = [
- "derive_builder",
+ "bon",
+ "caseless",
"entities",
- "http",
"memchr",
"once_cell",
"regex",
@@ -58,10 +108,25 @@ dependencies = [
]
[[package]]
-name = "darling"
-version = "0.20.9"
+name = "crc32fast"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
+[[package]]
+name = "darling"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
dependencies = [
"darling_core",
"darling_macro",
@@ -69,9 +134,9 @@ dependencies = [
[[package]]
name = "darling_core"
-version = "0.20.9"
+version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
dependencies = [
"fnv",
"ident_case",
@@ -83,9 +148,9 @@ dependencies = [
[[package]]
name = "darling_macro"
-version = "0.20.9"
+version = "0.20.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
@@ -93,48 +158,55 @@ dependencies = [
]
[[package]]
-name = "derive_builder"
-version = "0.20.0"
+name = "derive_arbitrary"
+version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
+checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
dependencies = [
- "derive_builder_macro",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
-dependencies = [
- "darling",
"proc-macro2",
"quote",
"syn",
]
-[[package]]
-name = "derive_builder_macro"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
-dependencies = [
- "derive_builder_core",
- "syn",
-]
-
[[package]]
name = "deunicode"
version = "1.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "322ef0094744e63628e6f0eb2295517f79276a5b342a4c2ff3042566ca181d4e"
+[[package]]
+name = "displaydoc"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "entities"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca"
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "flate2"
+version = "1.0.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
[[package]]
name = "fnv"
version = "1.0.7"
@@ -151,10 +223,16 @@ dependencies = [
]
[[package]]
-name = "heck"
-version = "0.4.1"
+name = "hashbrown"
+version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "http"
@@ -183,6 +261,22 @@ dependencies = [
"unicode-normalization",
]
+[[package]]
+name = "indexmap"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "inventory"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
+
[[package]]
name = "itoa"
version = "1.0.11"
@@ -218,18 +312,28 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
[[package]]
name = "libc"
version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+[[package]]
+name = "libloading"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
+dependencies = [
+ "cfg-if",
+ "windows-targets",
+]
+
+[[package]]
+name = "lockfree-object-pool"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
+
[[package]]
name = "log"
version = "0.4.21"
@@ -238,15 +342,24 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "memchr"
-version = "2.7.2"
+version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
+]
[[package]]
name = "once_cell"
-version = "1.19.0"
+version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
[[package]]
name = "percent-encoding"
@@ -260,17 +373,31 @@ version = "0.3.0"
dependencies = [
"base64",
"comrak",
+ "http",
"jemallocator",
+ "once_cell",
+ "regex",
"ring",
"rustler",
"url",
+ "zip",
+]
+
+[[package]]
+name = "prettyplease"
+version = "0.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033"
+dependencies = [
+ "proc-macro2",
+ "syn",
]
[[package]]
name = "proc-macro2"
-version = "1.0.80"
+version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e"
+checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
dependencies = [
"unicode-ident",
]
@@ -307,6 +434,12 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "regex-lite"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
+
[[package]]
name = "regex-syntax"
version = "0.8.3"
@@ -330,36 +463,40 @@ dependencies = [
[[package]]
name = "rustler"
-version = "0.28.0"
+version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7a2f98cb272ae0548e434bb3afde626012084cbebef84542bc7afed868bd6f"
+checksum = "b705f2c3643cc170d8888cb6bad589155d9c0248f3104ef7a04c2b7ffbaf13fc"
dependencies = [
- "lazy_static",
+ "inventory",
+ "libloading",
+ "regex-lite",
"rustler_codegen",
- "rustler_sys",
]
[[package]]
name = "rustler_codegen"
-version = "0.28.0"
+version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ae37fb154683a1ff13e95dfd16c15043d6aee9853ea650ca34a45bd94e319a5"
+checksum = "3ad56caff00562948bd6ac33c18dbc579e5a1bbee2d7f2f54073307e57f6b57a"
dependencies = [
"heck",
+ "inventory",
"proc-macro2",
"quote",
"syn",
]
[[package]]
-name = "rustler_sys"
-version = "2.2.1"
+name = "rustversion"
+version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae0eb19e2fdf4abc3662441087fc499a58b7550822cc8007e79f15c40157c883"
-dependencies = [
- "regex",
- "unreachable",
-]
+checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]]
name = "slug"
@@ -385,15 +522,35 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
-version = "2.0.59"
+version = "2.0.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a"
+checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
+[[package]]
+name = "thiserror"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "tinyvec"
version = "1.6.0"
@@ -442,15 +599,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
-[[package]]
-name = "unreachable"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
-dependencies = [
- "void",
-]
-
[[package]]
name = "untrusted"
version = "0.7.1"
@@ -468,12 +616,6 @@ dependencies = [
"percent-encoding",
]
-[[package]]
-name = "void"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
-
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
@@ -559,3 +701,98 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "zip"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494"
+dependencies = [
+ "arbitrary",
+ "crc32fast",
+ "crossbeam-utils",
+ "displaydoc",
+ "flate2",
+ "indexmap",
+ "memchr",
+ "thiserror",
+ "zopfli",
+]
+
+[[package]]
+name = "zopfli"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "lockfree-object-pool",
+ "log",
+ "once_cell",
+ "simd-adler32",
+]
diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml
index 942d39bc..e6bba775 100644
--- a/native/philomena/Cargo.toml
+++ b/native/philomena/Cargo.toml
@@ -10,12 +10,16 @@ path = "src/lib.rs"
crate-type = ["dylib"]
[dependencies]
-comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false }
-jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
-rustler = "0.28"
-ring = "0.16"
base64 = "0.21"
+comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false }
+http = "0.2"
+jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
+once_cell = "1.20"
+regex = "1"
+ring = "0.16"
+rustler = "0.35"
url = "2.3"
+zip = { version = "2.2.0", features = ["deflate"], default-features = false }
[profile.release]
opt-level = 3
diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs
index 5a72b3f6..c79f12e1 100644
--- a/native/philomena/src/camo.rs
+++ b/native/philomena/src/camo.rs
@@ -1,8 +1,8 @@
+use base64::engine::general_purpose::URL_SAFE_NO_PAD;
+use base64::Engine;
use ring::hmac;
use std::env;
use url::Url;
-use base64::Engine;
-use base64::engine::general_purpose::URL_SAFE_NO_PAD;
fn trusted_host(mut url: Url) -> Option {
url.set_port(Some(443)).ok()?;
@@ -11,7 +11,7 @@ fn trusted_host(mut url: Url) -> Option {
Some(url.to_string())
}
-fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option {
+fn untrusted_host(url: Url, camo_host: &str, camo_key: &str) -> Option {
let camo_url = format!("https://{}", camo_host);
let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref());
let tag = hmac::sign(&key, url.to_string().as_bytes());
@@ -27,20 +27,24 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option Option {
+pub fn image_url(uri: &str) -> Option {
let cdn_host = env::var("CDN_HOST").ok()?;
- let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| String::from(""));
- let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| String::from(""));
+ let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| "".into());
+ let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| "".into());
if camo_key.is_empty() {
- return Some(uri);
+ return Some(uri.into());
}
- let url = Url::parse(&uri).ok()?;
+ let url = Url::parse(uri).ok()?;
match url.host_str() {
Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url),
- Some(_) => untrusted_host(url, camo_host, camo_key),
- None => Some(String::from("")),
+ Some(_) => untrusted_host(url, &camo_host, &camo_key),
+ None => Some("".into()),
}
}
+
+pub fn image_url_careful(uri: &str) -> String {
+ image_url(uri).unwrap_or_else(|| "".into())
+}
diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs
new file mode 100644
index 00000000..9bcacfc9
--- /dev/null
+++ b/native/philomena/src/domains.rs
@@ -0,0 +1,38 @@
+use std::collections::BTreeSet;
+use std::env;
+
+use http::Uri;
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+pub type DomainSet = BTreeSet;
+
+static DOMAINS: Lazy