diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index a92aa72f..066732f5 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -34,6 +34,9 @@ jobs: - name: Build and test run: docker compose run app run-test + - name: mix format + run: docker compose run app mix format --check-formatted + - name: Security lint run: | docker compose run app mix sobelow --config @@ -50,6 +53,24 @@ jobs: - uses: actions/checkout@v4 - uses: crate-ci/typos@master + cargo: + name: Rust Linting and Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: cargo fmt + run: (cd native/philomena && cargo fmt --check) + + - name: cargo clippy + run: (cd native/philomena && cargo clippy -- -D warnings) + + - name: cargo test + run: (cd native/philomena && cargo test) + lint-and-test: name: 'JavaScript Linting and Unit Tests' runs-on: ubuntu-latest @@ -80,4 +101,4 @@ jobs: working-directory: ./assets - run: npm run build - working-directory: ./assets \ No newline at end of file + working-directory: ./assets diff --git a/.gitignore b/.gitignore index da2c70b6..0c221544 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,9 @@ npm-debug.log # VS Code .vscode -# ElixirLS +# Language server .elixir_ls +.lexical # Index dumps *.jsonl diff --git a/assets/js/__tests__/quick-tag.spec.ts b/assets/js/__tests__/quick-tag.spec.ts new file mode 100644 index 00000000..6b5d1ecd --- /dev/null +++ b/assets/js/__tests__/quick-tag.spec.ts @@ -0,0 +1,159 @@ +import { $, $$ } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { setupQuickTag } from '../quick-tag'; +import { fetchMock } from '../../test/fetch-mock.ts'; +import { waitFor } from '@testing-library/dom'; + +const quickTagData = `
+ Tag + + + +
+
+
+
+
+
+
+
+
`; + +describe('Batch tagging', () => { + let tagButton: HTMLAnchorElement; + let abortButton: HTMLAnchorElement; + let submitButton: HTMLAnchorElement; + let toggleAllButton: HTMLAnchorElement; + let mediaBoxes: HTMLDivElement[]; + + beforeEach(() => { + localStorage.clear(); + document.body.innerHTML = quickTagData; + + tagButton = assertNotNull($('.js-quick-tag')); + abortButton = assertNotNull($('.js-quick-tag--abort')); + submitButton = assertNotNull($('.js-quick-tag--submit')); + toggleAllButton = assertNotNull($('.js-quick-tag--all')); + mediaBoxes = $$('.media-box'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should prompt the user on click', () => { + const spy = vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).toContain('hidden'); + expect(abortButton.classList).not.toContain('hidden'); + expect(submitButton.classList).not.toContain('hidden'); + expect(toggleAllButton.classList).not.toContain('hidden'); + }); + + it('should not modify media boxes before entry', () => { + mediaBoxes[0].click(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + + it('should restore the list of tagged images on reload', () => { + // TODO: this is less than ideal, because it depends on the internal + // implementation of the quick-tag file. But we can't reload the page + // with jsdom. + localStorage.setItem('quickTagQueue', JSON.stringify(['0', '1'])); + localStorage.setItem('quickTagName', JSON.stringify('a')); + + setupQuickTag(); + expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected'); + }); + + describe('after entry', () => { + beforeEach(() => { + vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + }); + + it('should abort the tagging process if accepted', () => { + const spy = vi.spyOn(window, 'confirm').mockImplementation(() => true); + abortButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).not.toContain('hidden'); + expect(abortButton.classList).toContain('hidden'); + expect(submitButton.classList).toContain('hidden'); + expect(toggleAllButton.classList).toContain('hidden'); + }); + + it('should not abort the tagging process if rejected', () => { + const spy = vi.spyOn(window, 'confirm').mockImplementation(() => false); + abortButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).toContain('hidden'); + expect(abortButton.classList).not.toContain('hidden'); + expect(submitButton.classList).not.toContain('hidden'); + expect(toggleAllButton.classList).not.toContain('hidden'); + }); + + it('should toggle media box state on click', () => { + mediaBoxes[0].click(); + expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + + it('should toggle all media box states', () => { + mediaBoxes[0].click(); + toggleAllButton.click(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected'); + }); + }); + + describe('for submission', () => { + beforeAll(() => { + fetchMock.enableMocks(); + }); + + afterAll(() => { + fetchMock.disableMocks(); + }); + + beforeEach(() => { + vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + + fetchMock.resetMocks(); + mediaBoxes[0].click(); + mediaBoxes[1].click(); + }); + + it('should return to normal state on successful submission', () => { + fetchMock.mockResponse('{"failed":[]}'); + submitButton.click(); + + expect(fetch).toHaveBeenCalledOnce(); + + return waitFor(() => { + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + }); + + it('should show error on failed submission', () => { + fetchMock.mockResponse('{"failed":[0,1]}'); + submitButton.click(); + + const spy = vi.spyOn(window, 'alert').mockImplementation(() => {}); + + expect(fetch).toHaveBeenCalledOnce(); + + return waitFor(() => { + expect(spy).toHaveBeenCalledOnce(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + }); + }); +}); diff --git a/assets/js/__tests__/search.spec.ts b/assets/js/__tests__/search.spec.ts new file mode 100644 index 00000000..abc033b5 --- /dev/null +++ b/assets/js/__tests__/search.spec.ts @@ -0,0 +1,99 @@ +import { $ } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { setupSearch } from '../search'; +import { setupTagListener } from '../tagsinput'; + +const formData = `
+ + NOT + Numeric ID + My favorites + + +
`; + +describe('Search form help', () => { + beforeAll(() => { + setupSearch(); + setupTagListener(); + }); + + let input: HTMLInputElement; + let prependAnchor: HTMLAnchorElement; + let idAnchor: HTMLAnchorElement; + let favesAnchor: HTMLAnchorElement; + let helpNumeric: HTMLDivElement; + let subjectSpan: HTMLElement; + + beforeEach(() => { + document.body.innerHTML = formData; + + input = assertNotNull($('input')); + prependAnchor = assertNotNull($('a[data-search-prepend]')); + idAnchor = assertNotNull($('a[data-search-add="id.lte:10"]')); + favesAnchor = assertNotNull($('a[data-search-add="my:faves"]')); + helpNumeric = assertNotNull($('[data-search-help="numeric"]')); + subjectSpan = assertNotNull($('span', helpNumeric)); + }); + + it('should add text to input field', () => { + idAnchor.click(); + expect(input.value).toBe('id.lte:10'); + + favesAnchor.click(); + expect(input.value).toBe('id.lte:10, my:faves'); + }); + + it('should focus and select text in input field when requested', () => { + idAnchor.click(); + expect(input).toHaveFocus(); + expect(input.selectionStart).toBe(7); + expect(input.selectionEnd).toBe(9); + }); + + it('should highlight subject name when requested', () => { + expect(helpNumeric).toHaveClass('hidden'); + idAnchor.click(); + expect(helpNumeric).not.toHaveClass('hidden'); + expect(subjectSpan).toHaveTextContent('Numeric ID'); + }); + + it('should not focus and select text in input field when unavailable', () => { + favesAnchor.click(); + expect(input).not.toHaveFocus(); + expect(input.selectionStart).toBe(8); + expect(input.selectionEnd).toBe(8); + }); + + it('should not highlight subject name when unavailable', () => { + favesAnchor.click(); + expect(helpNumeric).toHaveClass('hidden'); + }); + + it('should prepend to empty input', () => { + prependAnchor.click(); + expect(input.value).toBe('-'); + }); + + it('should prepend to single input', () => { + input.value = 'a'; + prependAnchor.click(); + expect(input.value).toBe('-a'); + }); + + it('should prepend to comma-separated input', () => { + input.value = 'a,b'; + prependAnchor.click(); + expect(input.value).toBe('a,-b'); + }); + + it('should prepend to comma and space-separated input', () => { + input.value = 'a, b'; + prependAnchor.click(); + expect(input.value).toBe('a, -b'); + }); +}); diff --git a/assets/js/__tests__/tagsinput.spec.ts b/assets/js/__tests__/tagsinput.spec.ts new file mode 100644 index 00000000..e32b2dfc --- /dev/null +++ b/assets/js/__tests__/tagsinput.spec.ts @@ -0,0 +1,188 @@ +import { $, $$, hideEl } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { TermSuggestion } from '../utils/suggestions'; +import { setupTagsInput, addTag, reloadTagsInput } from '../tagsinput'; + +const formData = `
+
+ +
+ +
+
+ + + +
`; + +describe('Fancy tags input', () => { + let form: HTMLFormElement; + let tagBlock: HTMLDivElement; + let plainInput: HTMLTextAreaElement; + let fancyInput: HTMLDivElement; + let fancyText: HTMLInputElement; + let fancyShowButton: HTMLButtonElement; + let plainShowButton: HTMLButtonElement; + + beforeEach(() => { + window.booru.fancyTagUpload = true; + window.booru.fancyTagEdit = true; + document.body.innerHTML = formData; + + form = assertNotNull($('.tags-form')); + tagBlock = assertNotNull($('.js-tag-block')); + plainInput = assertNotNull($('.js-taginput-plain')); + fancyInput = assertNotNull($('.js-taginput-fancy')); + fancyText = assertNotNull($('.js-taginput-input')); + fancyShowButton = assertNotNull($('.js-taginput-show')); + plainShowButton = assertNotNull($('.js-taginput-hide')); + + // prevent these from submitting the form + fancyShowButton.addEventListener('click', e => e.preventDefault()); + plainShowButton.addEventListener('click', e => e.preventDefault()); + }); + + for (let i = 0; i < 4; i++) { + const type = (i & 2) === 0 ? 'upload' : 'edit'; + const name = (i & 2) === 0 ? 'fancyTagUpload' : 'fancyTagEdit'; + const value = (i & 1) === 0; + + // eslint-disable-next-line no-loop-func + it(`should imply ${name}:${value} <-> ${type}:${value} on setup`, () => { + window.booru.fancyTagEdit = false; + window.booru.fancyTagUpload = false; + window.booru[name] = value; + + plainInput.value = 'a, b'; + tagBlock.classList.remove('fancy-tag-edit', 'fancy-tag-upload'); + tagBlock.classList.add(`fancy-tag-${type}`); + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + expect($$('span.tag', fancyInput)).toHaveLength(value ? 2 : 0); + }); + } + + it('should move tags from the plain to the fancy editor when the fancy editor is shown', () => { + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + plainInput.value = 'a, b'; + fancyShowButton.click(); + expect($$('span.tag', fancyInput)).toHaveLength(2); + }); + + it('should move tags from the plain to the fancy editor on reload event', () => { + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + plainInput.value = 'a, b'; + reloadTagsInput(plainInput); + expect($$('span.tag', fancyInput)).toHaveLength(2); + }); + + it('should respond to addtag events', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should not respond to addtag events if the container is hidden', () => { + setupTagsInput(tagBlock); + hideEl(fancyInput); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should respond to autocomplete events', () => { + setupTagsInput(tagBlock); + fancyText.dispatchEvent(new CustomEvent('autocomplete', { detail: { value: 'a', label: 'a' } })); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should allow removing previously added tags by clicking them', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + assertNotNull($('span.tag a', fancyInput)).click(); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should allow removing previously added tags by adding one with a minus sign prepended', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + addTag(plainInput, '-a'); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should disallow adding empty tags', () => { + setupTagsInput(tagBlock); + addTag(plainInput, ''); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should disallow adding existing tags', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should submit the form on ctrl+enter', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 13, ctrlKey: true, bubbles: true }); + + return new Promise(resolve => { + form.addEventListener('submit', e => { + e.preventDefault(); + resolve(); + }); + + fancyText.dispatchEvent(ev); + expect(ev.defaultPrevented).toBe(true); + }); + }); + + it('does nothing when backspacing on empty input and there are no tags', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true }); + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('erases the last added tag when backspacing on empty input', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + addTag(plainInput, 'b'); + + const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true }); + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('adds new tag when comma is pressed', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 188, bubbles: true }); + fancyText.value = 'a'; + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + expect(fancyText.value).toBe(''); + }); + + it('adds new tag when enter is pressed', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 13, bubbles: true }); + fancyText.value = 'a'; + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + expect(fancyText.value).toBe(''); + }); +}); diff --git a/assets/js/autocomplete.ts b/assets/js/autocomplete.ts index 489392c3..61340b48 100644 --- a/assets/js/autocomplete.ts +++ b/assets/js/autocomplete.ts @@ -25,6 +25,14 @@ function restoreOriginalValue() { if (isSearchField(inputField) && originalQuery) { inputField.value = originalQuery; + + if (selectedTerm) { + const [, selectedTermEnd] = selectedTerm[0]; + + inputField.setSelectionRange(selectedTermEnd, selectedTermEnd); + } + + return; } if (originalTerm) { diff --git a/assets/js/markdowntoolbar.ts b/assets/js/markdowntoolbar.ts index f9ceb840..0b355e85 100644 --- a/assets/js/markdowntoolbar.ts +++ b/assets/js/markdowntoolbar.ts @@ -60,7 +60,7 @@ const markdownSyntax: Record = { }, subscript: { action: wrapSelection, - options: { prefix: '%' }, + options: { prefix: '~' }, }, quote: { action: wrapLines, diff --git a/assets/js/quick-tag.js b/assets/js/quick-tag.js deleted file mode 100644 index 4457784a..00000000 --- a/assets/js/quick-tag.js +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Quick Tag - */ - -import store from './utils/store'; -import { $, $$, toggleEl, onLeftClick } from './utils/dom'; -import { fetchJson, handleError } from './utils/requests'; - -const imageQueueStorage = 'quickTagQueue'; -const currentTagStorage = 'quickTagName'; - -function currentQueue() { - return store.get(imageQueueStorage) || []; -} - -function currentTags() { - return store.get(currentTagStorage) || ''; -} - -function getTagButton() { - return $('.js-quick-tag'); -} - -function setTagButton(text) { - $('.js-quick-tag--submit span').textContent = text; -} - -function toggleActiveState() { - toggleEl($('.js-quick-tag'), $('.js-quick-tag--abort'), $('.js-quick-tag--all'), $('.js-quick-tag--submit')); - - setTagButton(`Submit (${currentTags()})`); - - $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected')); - $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected')); - currentQueue().forEach(id => - $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')), - ); -} - -function activate() { - store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:')); - - if (currentTags()) toggleActiveState(); -} - -function reset() { - store.remove(currentTagStorage); - store.remove(imageQueueStorage); - - toggleActiveState(); -} - -function promptReset() { - if (window.confirm('Are you sure you want to abort batch tagging?')) { - reset(); - } -} - -function submit() { - setTagButton(`Wait... (${currentTags()})`); - - fetchJson('PUT', '/admin/batch/tags', { - tags: currentTags(), - image_ids: currentQueue(), - }) - .then(handleError) - .then(r => r.json()) - .then(data => { - if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`); - - reset(); - }); -} - -function modifyImageQueue(mediaBox) { - if (currentTags()) { - const imageId = mediaBox.dataset.imageId; - const queue = currentQueue(); - const isSelected = queue.includes(imageId); - - isSelected ? queue.splice(queue.indexOf(imageId), 1) : queue.push(imageId); - - $$(`.media-box__header[data-image-id="${imageId}"]`).forEach(el => - el.classList.toggle('media-box__header--selected'), - ); - - store.set(imageQueueStorage, queue); - } -} - -function toggleAllImages() { - $$('#imagelist-container .media-box').forEach(modifyImageQueue); -} - -function clickHandler(event) { - const targets = { - '.js-quick-tag': activate, - '.js-quick-tag--abort': promptReset, - '.js-quick-tag--submit': submit, - '.js-quick-tag--all': toggleAllImages, - '.media-box': modifyImageQueue, - }; - - for (const target in targets) { - if (event.target && event.target.closest(target)) { - targets[target](event.target.closest(target)); - currentTags() && event.preventDefault(); - } - } -} - -function setupQuickTag() { - if (getTagButton() && currentTags()) toggleActiveState(); - if (getTagButton()) onLeftClick(clickHandler); -} - -export { setupQuickTag }; diff --git a/assets/js/quick-tag.ts b/assets/js/quick-tag.ts new file mode 100644 index 00000000..3d462cb2 --- /dev/null +++ b/assets/js/quick-tag.ts @@ -0,0 +1,124 @@ +/** + * Quick Tag + */ + +import store from './utils/store'; +import { assertNotNull, assertNotUndefined } from './utils/assert'; +import { $, $$, toggleEl } from './utils/dom'; +import { fetchJson, handleError } from './utils/requests'; +import { delegate, leftClick } from './utils/events'; + +const imageQueueStorage = 'quickTagQueue'; +const currentTagStorage = 'quickTagName'; + +function currentQueue(): string[] { + return store.get(imageQueueStorage) || []; +} + +function currentTags(): string { + return store.get(currentTagStorage) || ''; +} + +function setTagButton(text: string) { + assertNotNull($('.js-quick-tag--submit span')).textContent = text; +} + +function toggleActiveState() { + toggleEl($$('.js-quick-tag,.js-quick-tag--abort,.js-quick-tag--all,.js-quick-tag--submit')); + + setTagButton(`Submit (${currentTags()})`); + + $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected')); + $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected')); + + currentQueue().forEach(id => + $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')), + ); +} + +function activate(event: Event) { + event.preventDefault(); + + store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:')); + + if (currentTags()) { + toggleActiveState(); + } +} + +function reset() { + store.remove(currentTagStorage); + store.remove(imageQueueStorage); + + toggleActiveState(); +} + +function promptReset(event: Event) { + event.preventDefault(); + + if (window.confirm('Are you sure you want to abort batch tagging?')) { + reset(); + } +} + +function submit(event: Event) { + event.preventDefault(); + + setTagButton(`Wait... (${currentTags()})`); + + fetchJson('PUT', '/admin/batch/tags', { + tags: currentTags(), + image_ids: currentQueue(), + }) + .then(handleError) + .then(r => r.json()) + .then(data => { + if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`); + + reset(); + }); +} + +function modifyImageQueue(event: Event, mediaBox: HTMLDivElement) { + if (!currentTags()) { + return; + } + + const imageId = assertNotUndefined(mediaBox.dataset.imageId); + const queue = currentQueue(); + const isSelected = queue.includes(imageId); + + if (isSelected) { + queue.splice(queue.indexOf(imageId), 1); + } else { + queue.push(imageId); + } + + for (const boxHeader of $$(`.media-box__header[data-image-id="${imageId}"]`)) { + boxHeader.classList.toggle('media-box__header--selected'); + } + + store.set(imageQueueStorage, queue); + event.preventDefault(); +} + +function toggleAllImages(event: Event, _target: Element) { + for (const mediaBox of $$('#imagelist-container .media-box')) { + modifyImageQueue(event, mediaBox); + } +} + +delegate(document, 'click', { + '.js-quick-tag': leftClick(activate), + '.js-quick-tag--abort': leftClick(promptReset), + '.js-quick-tag--submit': leftClick(submit), + '.js-quick-tag--all': leftClick(toggleAllImages), + '.media-box': leftClick(modifyImageQueue), +}); + +export function setupQuickTag() { + const tagButton = $('.js-quick-tag'); + if (tagButton && currentTags()) { + toggleActiveState(); + } +} diff --git a/assets/js/search.ts b/assets/js/search.ts index 269497d2..eff8d98a 100644 --- a/assets/js/search.ts +++ b/assets/js/search.ts @@ -1,54 +1,85 @@ -import { $, $$ } from './utils/dom'; +import { assertNotNull, assertNotUndefined } from './utils/assert'; +import { $, $$, showEl, hideEl } from './utils/dom'; +import { delegate, leftClick } from './utils/events'; import { addTag } from './tagsinput'; -function showHelp(subject: string, type: string | null) { - $$('[data-search-help]').forEach(helpBox => { - if (helpBox.getAttribute('data-search-help') === type) { - const searchSubject = $('.js-search-help-subject', helpBox); - - if (searchSubject) { - searchSubject.textContent = subject; - } - - helpBox.classList.remove('hidden'); - } else { - helpBox.classList.add('hidden'); - } - }); -} - -function prependToLast(field: HTMLInputElement, value: string) { - const separatorIndex = field.value.lastIndexOf(','); - const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1; - field.value = - field.value.slice(0, separatorIndex + advanceBy) + value + field.value.slice(separatorIndex + advanceBy); -} - -function selectLast(field: HTMLInputElement, characterCount: number) { +function focusAndSelectLast(field: HTMLInputElement, characterCount: number) { field.focus(); - field.selectionStart = field.value.length - characterCount; field.selectionEnd = field.value.length; } -function executeFormHelper(e: PointerEvent) { - if (!e.target) { - return; - } +function prependToLast(field: HTMLInputElement, value: string) { + // Find the last comma in the input and advance past it + const separatorIndex = field.value.lastIndexOf(','); + const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1; - const searchField = $('.js-search-field'); - const attr = (name: string) => e.target && (e.target as HTMLElement).getAttribute(name); + // Insert the value string at the new location + field.value = [ + field.value.slice(0, separatorIndex + advanceBy), + value, + field.value.slice(separatorIndex + advanceBy), + ].join(''); +} - if (attr('data-search-add')) addTag(searchField, attr('data-search-add')); - if (attr('data-search-show-help')) showHelp((e.target as Node).textContent || '', attr('data-search-show-help')); - if (attr('data-search-select-last') && searchField) { - selectLast(searchField, parseInt(attr('data-search-select-last') || '', 10)); +function getAssociatedData(target: HTMLElement) { + const form = assertNotNull(target.closest('form')); + const input = assertNotNull($('.js-search-field', form)); + const helpBoxes = $$('[data-search-help]', form); + + return { input, helpBoxes }; +} + +function showHelp(helpBoxes: HTMLDivElement[], typeName: string, subject: string) { + for (const helpBox of helpBoxes) { + // Get the subject name span + const subjectName = assertNotNull($('.js-search-help-subject', helpBox)); + + // Take the appropriate action for this help box + if (helpBox.dataset.searchHelp === typeName) { + subjectName.textContent = subject; + showEl(helpBox); + } else { + hideEl(helpBox); + } } - if (attr('data-search-prepend') && searchField) prependToLast(searchField, attr('data-search-prepend') || ''); +} + +function onSearchAdd(_event: Event, target: HTMLAnchorElement) { + // Load form + const { input, helpBoxes } = getAssociatedData(target); + + // Get data for this link + const addValue = assertNotUndefined(target.dataset.searchAdd); + const showHelpValue = assertNotUndefined(target.dataset.searchShowHelp); + const selectLastValue = target.dataset.searchSelectLast; + + // Add the tag + addTag(input, addValue); + + // Show associated help, if available + showHelp(helpBoxes, showHelpValue, assertNotNull(target.textContent)); + + // Select last characters, if requested + if (selectLastValue) { + focusAndSelectLast(input, Number(selectLastValue)); + } +} + +function onSearchPrepend(_event: Event, target: HTMLAnchorElement) { + // Load form + const { input } = getAssociatedData(target); + + // Get data for this link + const prependValue = assertNotUndefined(target.dataset.searchPrepend); + + // Prepend + prependToLast(input, prependValue); } export function setupSearch() { - const form = $('.js-search-form'); - - if (form) form.addEventListener('click', executeFormHelper as EventListener); + delegate(document, 'click', { + 'form.js-search-form a[data-search-add][data-search-show-help]': leftClick(onSearchAdd), + 'form.js-search-form a[data-search-prepend]': leftClick(onSearchPrepend), + }); } diff --git a/assets/js/tagsinput.js b/assets/js/tagsinput.ts similarity index 63% rename from assets/js/tagsinput.js rename to assets/js/tagsinput.ts index bc05b3d7..74574906 100644 --- a/assets/js/tagsinput.js +++ b/assets/js/tagsinput.ts @@ -2,14 +2,20 @@ * Fancy tag editor. */ +import { assertNotNull, assertType } from './utils/assert'; import { $, $$, clearEl, removeEl, showEl, hideEl, escapeCss, escapeHtml } from './utils/dom'; +import { TermSuggestion } from './utils/suggestions'; -function setupTagsInput(tagBlock) { - const [textarea, container] = $$('.js-taginput', tagBlock); - const setup = $('.js-tag-block ~ button', tagBlock.parentNode); - const inputField = $('input', container); +export function setupTagsInput(tagBlock: HTMLDivElement) { + const form = assertNotNull(tagBlock.closest('form')); + const textarea = assertNotNull($('.js-taginput-plain', tagBlock)); + const container = assertNotNull($('.js-taginput-fancy', tagBlock)); + const parentField = assertNotNull(tagBlock.parentElement); + const setup = assertNotNull($('.js-tag-block ~ button', parentField)); + const inputField = assertNotNull($('input', container)); + const submitButton = assertNotNull($('[type="submit"]', form)); - let tags = []; + let tags: string[] = []; // Load in the current tag set from the textarea setup.addEventListener('click', importTags); @@ -27,7 +33,7 @@ function setupTagsInput(tagBlock) { inputField.addEventListener('keydown', handleKeyEvent); // Respond to autocomplete form clicks - inputField.addEventListener('autocomplete', handleAutocomplete); + inputField.addEventListener('autocomplete', handleAutocomplete as EventListener); // Respond to Ctrl+Enter shortcut tagBlock.addEventListener('keydown', handleCtrlEnter); @@ -35,19 +41,19 @@ function setupTagsInput(tagBlock) { // TODO: Cleanup this bug fix // Switch to fancy tagging if user settings want it if (fancyEditorRequested(tagBlock)) { - showEl($$('.js-taginput-fancy')); - showEl($$('.js-taginput-hide')); - hideEl($$('.js-taginput-plain')); - hideEl($$('.js-taginput-show')); + showEl($$('.js-taginput-fancy')); + showEl($$('.js-taginput-hide')); + hideEl($$('.js-taginput-plain')); + hideEl($$('.js-taginput-show')); importTags(); } - function handleAutocomplete(event) { + function handleAutocomplete(event: CustomEvent) { insertTag(event.detail.value); inputField.focus(); } - function handleAddTag(event) { + function handleAddTag(event: AddtagEvent) { // Ignore if not in tag edit mode if (container.classList.contains('hidden')) return; @@ -55,14 +61,16 @@ function setupTagsInput(tagBlock) { event.stopPropagation(); } - function handleTagClear(event) { - if (event.target.dataset.tagName) { + function handleTagClear(event: Event) { + const target = assertType(event.target, HTMLElement); + + if (target.dataset.tagName) { event.preventDefault(); - removeTag(event.target.dataset.tagName, event.target.parentNode); + removeTag(target.dataset.tagName, assertNotNull(target.parentElement)); } } - function handleKeyEvent(event) { + function handleKeyEvent(event: KeyboardEvent) { const { keyCode, ctrlKey, shiftKey } = event; // allow form submission with ctrl+enter if no text was typed @@ -73,7 +81,7 @@ function setupTagsInput(tagBlock) { // backspace on a blank input field if (keyCode === 8 && inputField.value === '') { event.preventDefault(); - const erased = $('.tag:last-of-type', container); + const erased = $('.tag:last-of-type', container); if (erased) removeTag(tags[tags.length - 1], erased); } @@ -86,14 +94,14 @@ function setupTagsInput(tagBlock) { } } - function handleCtrlEnter(event) { + function handleCtrlEnter(event: KeyboardEvent) { const { keyCode, ctrlKey } = event; if (keyCode !== 13 || !ctrlKey) return; - $('[type="submit"]', tagBlock.closest('form')).click(); + submitButton.click(); } - function insertTag(name) { + function insertTag(name: string) { name = name.trim(); // eslint-disable-line no-param-reassign // Add if not degenerate or already present @@ -102,9 +110,9 @@ function setupTagsInput(tagBlock) { // Remove instead if the tag name starts with a minus if (name[0] === '-') { name = name.slice(1); // eslint-disable-line no-param-reassign - const tagLink = $(`[data-tag-name="${escapeCss(name)}"]`, container); + const tagLink = assertNotNull($(`[data-tag-name="${escapeCss(name)}"]`, container)); - return removeTag(name, tagLink.parentNode); + return removeTag(name, assertNotNull(tagLink.parentElement)); } tags.push(name); @@ -116,7 +124,7 @@ function setupTagsInput(tagBlock) { inputField.value = ''; } - function removeTag(name, element) { + function removeTag(name: string, element: HTMLElement) { removeEl(element); // Remove the tag from the list @@ -134,7 +142,7 @@ function setupTagsInput(tagBlock) { } } -function fancyEditorRequested(tagBlock) { +function fancyEditorRequested(tagBlock: HTMLDivElement) { // Check whether the user made the fancy editor the default for each type of tag block. return ( (window.booru.fancyTagUpload && tagBlock.classList.contains('fancy-tag-upload')) || @@ -142,19 +150,17 @@ function fancyEditorRequested(tagBlock) { ); } -function setupTagListener() { +export function setupTagListener() { document.addEventListener('addtag', event => { if (event.target.value) event.target.value += ', '; event.target.value += event.detail.name; }); } -function addTag(textarea, name) { +export function addTag(textarea: HTMLInputElement | HTMLTextAreaElement, name: string) { textarea.dispatchEvent(new CustomEvent('addtag', { detail: { name }, bubbles: true })); } -function reloadTagsInput(textarea) { +export function reloadTagsInput(textarea: HTMLInputElement | HTMLTextAreaElement) { textarea.dispatchEvent(new CustomEvent('reload')); } - -export { setupTagsInput, setupTagListener, addTag, reloadTagsInput }; diff --git a/assets/package-lock.json b/assets/package-lock.json index 0f73ac68..3024e2ec 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -2380,9 +2380,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", diff --git a/assets/test/vitest-setup.ts b/assets/test/vitest-setup.ts index 27688447..d533bd4d 100644 --- a/assets/test/vitest-setup.ts +++ b/assets/test/vitest-setup.ts @@ -7,6 +7,8 @@ import { fireEvent } from '@testing-library/dom'; window.booru = { timeAgo: () => {}, csrfToken: 'mockCsrfToken', + fancyTagEdit: true, + fancyTagUpload: true, hiddenTag: '/mock-tagblocked.svg', hiddenTagList: [], hideStaffTools: 'true', diff --git a/assets/types/booru-object.d.ts b/assets/types/booru-object.d.ts index 4154385c..8b04cced 100644 --- a/assets/types/booru-object.d.ts +++ b/assets/types/booru-object.d.ts @@ -73,6 +73,14 @@ interface BooruObject { * List of image IDs in the current gallery. */ galleryImages?: number[]; + /** + * Fancy tag setting for uploading images. + */ + fancyTagUpload: boolean; + /** + * Fancy tag setting for editing images. + */ + fancyTagEdit: boolean; } declare global { diff --git a/assets/types/tags.ts b/assets/types/tags.ts new file mode 100644 index 00000000..b132dbf0 --- /dev/null +++ b/assets/types/tags.ts @@ -0,0 +1,20 @@ +export {}; + +declare global { + interface Addtag { + name: string; + } + + interface AddtagEvent extends CustomEvent { + target: HTMLInputElement | HTMLTextAreaElement; + } + + interface ReloadEvent extends CustomEvent { + target: HTMLInputElement | HTMLTextAreaElement; + } + + interface GlobalEventHandlersEventMap { + addtag: AddtagEvent; + reload: ReloadEvent; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 5b102408..a1233418 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,16 +64,14 @@ services: - POSTGRES_PASSWORD=postgres volumes: - postgres_data:/var/lib/postgresql/data - logging: - driver: "none" + attach: false opensearch: image: opensearchproject/opensearch:2.16.0 volumes: - opensearch_data:/usr/share/opensearch/data - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml - logging: - driver: "none" + attach: false ulimits: nofile: soft: 65536 @@ -81,8 +79,7 @@ services: valkey: image: valkey/valkey:8.0-alpine - logging: - driver: "none" + attach: false files: image: andrewgaul/s3proxy:sha-4976e17 @@ -90,6 +87,7 @@ services: - JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3 volumes: - .:/srv/philomena + attach: false web: build: @@ -106,8 +104,7 @@ services: environment: - AWS_ACCESS_KEY_ID=local-identity - AWS_SECRET_ACCESS_KEY=local-credential - logging: - driver: "none" + attach: false depends_on: - app ports: diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex index 87ab48d3..61c61f87 100644 --- a/lib/mix/tasks/upload_to_s3.ex +++ b/lib/mix/tasks/upload_to_s3.ex @@ -121,7 +121,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name), timeout: :infinity @@ -142,7 +144,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_images(queryable, batch_size, file_root, new_file_root) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity) |> Stream.run() diff --git a/lib/philomena/data_exports/aggregator.ex b/lib/philomena/data_exports/aggregator.ex new file mode 100644 index 00000000..5dcb82b3 --- /dev/null +++ b/lib/philomena/data_exports/aggregator.ex @@ -0,0 +1,171 @@ +defmodule Philomena.DataExports.Aggregator do + @moduledoc """ + Data generation module for data export logic. + """ + + import Ecto.Query + alias PhilomenaQuery.Batch + + # Direct PII + alias Philomena.Donations.Donation + alias Philomena.UserFingerprints.UserFingerprint + alias Philomena.UserIps.UserIp + alias Philomena.UserNameChanges.UserNameChange + alias Philomena.Users.User + + # UGC for export + alias Philomena.ArtistLinks.ArtistLink + alias Philomena.Badges.Award + alias Philomena.Comments.Comment + alias Philomena.Commissions.Commission + alias Philomena.DnpEntries.DnpEntry + alias Philomena.DuplicateReports.DuplicateReport + alias Philomena.Filters.Filter + alias Philomena.ImageFaves.ImageFave + alias Philomena.ImageHides.ImageHide + alias Philomena.ImageVotes.ImageVote + alias Philomena.Images.Image + alias Philomena.PollVotes.PollVote + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.SourceChanges.SourceChange + alias Philomena.TagChanges.TagChange + alias Philomena.Topics.Topic + alias Philomena.Bans.User, as: UserBan + + # Direct UGC from form submission + @user_columns [ + :created_at, + :name, + :email, + :description, + :current_filter_id, + :spoiler_type, + :theme, + :images_per_page, + :show_large_thumbnails, + :show_sidebar_and_watched_images, + :fancy_tag_field_on_upload, + :fancy_tag_field_on_edit, + :fancy_tag_field_in_settings, + :autorefresh_by_default, + :anonymous_by_default, + :comments_newest_first, + :comments_always_jump_to_last, + :comments_per_page, + :watch_on_reply, + :watch_on_new_topic, + :watch_on_upload, + :messages_newest_first, + :serve_webm, + :no_spoilered_in_watched, + :watched_images_query_str, + :watched_images_exclude_str, + :use_centered_layout, + :personal_title, + :hide_vote_counts, + :scale_large_images + ] + + # All these also have created_at and are selected by user_id + @indirect_columns [ + {Donation, [:email, :amount, :fee, :note]}, + {UserFingerprint, [:fingerprint, :uses, :updated_at]}, + {UserIp, [:ip, :uses, :updated_at]}, + {UserNameChange, [:name]}, + {ArtistLink, [:aasm_state, :uri, :public, :tag_id]}, + {Award, [:label, :badge_name, :badge_id]}, + {Comment, + [ + :ip, + :fingerprint, + :user_agent, + :referrer, + :anonymous, + :image_id, + :edited_at, + :edit_reason, + :body + ]}, + {Commission, + [:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]}, + {DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions], + :requesting_user_id}, + {DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]}, + {Filter, + [ + :name, + :description, + :public, + :hidden_complex_str, + :spoilered_complex_str, + :hidden_tag_ids, + :spoilered_tag_ids + ]}, + {ImageFave, [:image_id], :user_id, :image_id}, + {ImageHide, [:image_id], :user_id, :image_id}, + {ImageVote, [:image_id, :up], :user_id, :image_id}, + {Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]}, + {PollVote, [:rank, :poll_option_id]}, + {Post, + [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]}, + {Report, + [:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]}, + {SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]}, + {TagChange, + [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]}, + {Topic, [:title, :anonymous, :forum_id]}, + {UserBan, [:reason, :generated_ban_id]} + ] + + @doc """ + Get all of the export data for the given user. + """ + def get_for_user(user_id) do + [select_user(user_id)] ++ select_indirect(user_id) + end + + defp select_user(user_id) do + select_schema_by_key(user_id, User, @user_columns, :id) + end + + defp select_indirect(user_id) do + Enum.map(@indirect_columns, fn + {schema_name, columns} -> + select_schema_by_key(user_id, schema_name, columns) + + {schema_name, columns, key_column} -> + select_schema_by_key(user_id, schema_name, columns, key_column) + + {schema_name, columns, key_column, id_field} -> + select_schema_by_key(user_id, schema_name, columns, key_column, id_field) + end) + end + + defp select_schema_by_key( + user_id, + schema_name, + columns, + key_column \\ :user_id, + id_field \\ :id + ) do + table_name = schema_name.__schema__(:source) + columns = [:created_at] ++ columns + + {"#{table_name}.jsonl", + schema_name + |> where([s], field(s, ^key_column) == ^user_id) + |> select([s], map(s, ^columns)) + |> Batch.records(id_field: id_field) + |> results_as_json_lines()} + end + + defp results_as_json_lines(list_of_maps) do + Stream.map(list_of_maps, fn map -> + map + |> Map.new(fn {k, v} -> {k, to_string(v)} end) + |> Jason.encode!() + |> Kernel.<>("\n") + end) + end +end diff --git a/lib/philomena/data_exports/zip_generator.ex b/lib/philomena/data_exports/zip_generator.ex new file mode 100644 index 00000000..8c5aaf7e --- /dev/null +++ b/lib/philomena/data_exports/zip_generator.ex @@ -0,0 +1,56 @@ +defmodule Philomena.DataExports.ZipGenerator do + @moduledoc """ + ZIP file generator for an export. + """ + + alias Philomena.Native + + @doc """ + Write the ZIP file for the given aggregate data. + + Expects a list of 2-tuples, with the first element being the name of the + file to generate, and the second element being a stream which generates the + binary contents of the file. + """ + @spec generate(Path.t(), Enumerable.t()) :: :ok | atom() + def generate(filename, aggregate) do + case Native.zip_open_writer(filename) do + {:ok, zip} -> + stream_aggregate(zip, aggregate) + + error -> + error + end + end + + @spec stream_aggregate(reference(), Enumerable.t()) :: {:ok, reference()} | :error + defp stream_aggregate(zip, aggregate) do + aggregate + |> Enum.reduce_while(:ok, fn {name, content_stream}, _ -> + with :ok <- Native.zip_start_file(zip, name), + :ok <- stream_file_data(zip, content_stream) do + {:cont, :ok} + else + error -> + {:halt, error} + end + end) + |> case do + :ok -> + Native.zip_finish(zip) + + error -> + error + end + end + + @spec stream_file_data(reference(), Enumerable.t(iodata())) :: :ok | :error + defp stream_file_data(zip, content_stream) do + Enum.reduce_while(content_stream, :ok, fn iodata, _ -> + case Native.zip_write(zip, IO.iodata_to_binary(iodata)) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end +end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index a67dc33f..14eeaa17 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -11,4 +11,16 @@ defmodule Philomena.Native do @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()} + def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_start_file(reference(), String.t()) :: :ok | :error + def zip_start_file(_zip, _name), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_write(reference(), binary()) :: :ok | :error + def zip_write(_zip, _data), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_finish(reference()) :: :ok | :error + def zip_finish(_zip), do: :erlang.nif_error(:nif_not_loaded) end diff --git a/lib/philomena/user_downvote_wipe.ex b/lib/philomena/user_downvote_wipe.ex index 6fb217fe..b958ab9b 100644 --- a/lib/philomena/user_downvote_wipe.ex +++ b/lib/philomena/user_downvote_wipe.ex @@ -15,7 +15,8 @@ defmodule Philomena.UserDownvoteWipe do ImageVote |> where(user_id: ^user.id, up: false) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -31,7 +32,8 @@ defmodule Philomena.UserDownvoteWipe do if upvotes_and_faves_too do ImageVote |> where(user_id: ^user.id, up: true) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -46,7 +48,8 @@ defmodule Philomena.UserDownvoteWipe do ImageFave |> where(user_id: ^user.id) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_f], i_f.image_id)) {count, nil} = diff --git a/lib/philomena/workers/tag_change_revert_worker.ex b/lib/philomena/workers/tag_change_revert_worker.ex index 519b8404..80058a31 100644 --- a/lib/philomena/workers/tag_change_revert_worker.ex +++ b/lib/philomena/workers/tag_change_revert_worker.ex @@ -27,7 +27,9 @@ defmodule Philomena.TagChangeRevertWorker do batch_size = attributes["batch_size"] || 100 attributes = Map.delete(attributes, "batch_size") - Batch.query_batches(queryable, [batch_size: batch_size], fn queryable -> + queryable + |> Batch.query_batches(batch_size: batch_size) + |> Enum.each(fn queryable -> ids = Repo.all(select(queryable, [tc], tc.id)) TagChanges.mass_revert(ids, cast_ip(atomify_keys(attributes))) end) diff --git a/lib/philomena_proxy/scrapers.ex b/lib/philomena_proxy/scrapers.ex index a96f0817..08674d44 100644 --- a/lib/philomena_proxy/scrapers.ex +++ b/lib/philomena_proxy/scrapers.ex @@ -21,6 +21,7 @@ defmodule PhilomenaProxy.Scrapers do } @scrapers [ + PhilomenaProxy.Scrapers.Bluesky, PhilomenaProxy.Scrapers.Deviantart, PhilomenaProxy.Scrapers.Pillowfort, PhilomenaProxy.Scrapers.Twitter, diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex new file mode 100644 index 00000000..e67672b2 --- /dev/null +++ b/lib/philomena_proxy/scrapers/bluesky.ex @@ -0,0 +1,76 @@ +defmodule PhilomenaProxy.Scrapers.Bluesky do + @moduledoc false + + alias PhilomenaProxy.Scrapers.Scraper + alias PhilomenaProxy.Scrapers + + @behaviour Scraper + + @url_regex ~r|https://bsky\.app/profile/([^/]+)/post/([^/?#]+)| + @fullsize_image_regex ~r|.*/img/feed_fullsize/plain/([^/]+)/([^@]+).*| + @blob_image_url_pattern "https://bsky.social/xrpc/com.atproto.sync.getBlob/?did=\\1&cid=\\2" + + @spec can_handle?(URI.t(), String.t()) :: boolean() + def can_handle?(_uri, url) do + String.match?(url, @url_regex) + end + + @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result() + def scrape(_uri, url) do + [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) + + did = fetch_did(handle) + + api_url_get_posts = + "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" + + post_json = + api_url_get_posts + |> PhilomenaProxy.Http.get() + |> json!() + |> Map.fetch!("posts") + |> hd() + + %{ + source_url: url, + author_name: domain_first_component(post_json["author"]["handle"]), + description: post_json["record"]["text"], + images: + Enum.map( + post_json["embed"]["images"], + &%{ + url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern), + camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"]) + } + ) + } + end + + defp fetch_did(handle) do + case handle do + <<"did:", _rest::binary>> -> + handle + + _ -> + api_url_resolve_handle = + "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" + + api_url_resolve_handle + |> PhilomenaProxy.Http.get() + |> json!() + |> Map.fetch!("did") + end + end + + defp domain_first_component(domain) do + case String.split(domain, ".") do + [name | _] -> + name + + _ -> + domain + end + end + + defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body) +end diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex index 918a3b5e..bb9c2c74 100644 --- a/lib/philomena_query/batch.ex +++ b/lib/philomena_query/batch.ex @@ -25,24 +25,32 @@ defmodule PhilomenaQuery.Batch do @type id_field :: {:id_field, atom()} @type batch_options :: [batch_size() | id_field()] - @typedoc """ - The callback for `record_batches/3`. + @doc """ + Stream schema structures on a queryable, using batches to avoid locking. + + Valid options: + * `batch_size` (integer) - the number of records to load per batch + * `id_field` (atom) - the name of the field containing the ID + + ## Example + + queryable = from i in Image, where: i.image_width >= 1920 + + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(fn image -> IO.inspect(image.id) end) - Takes a list of schema structs which were returned in the batch. Return value is ignored. """ - @type record_batch_callback :: ([struct()] -> any()) - - @typedoc """ - The callback for `query_batches/3`. - - Takes an `m:Ecto.Query` that can be processed with `m:Philomena.Repo` query commands, such - as `Philomena.Repo.update_all/3` or `Philomena.Repo.delete_all/2`. Return value is ignored. - """ - @type query_batch_callback :: ([Ecto.Query.t()] -> any()) + @spec records(queryable(), batch_options()) :: Enumerable.t() + def records(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.flat_map(&Repo.all/1) + end @doc """ - Execute a callback with lists of schema structures on a queryable, - using batches to avoid locking. + Stream lists of schema structures on a queryable, using batches to avoid + locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -56,16 +64,20 @@ defmodule PhilomenaQuery.Batch do Enum.each(images, &IO.inspect(&1.id)) end - PhilomenaQuery.Batch.record_batches(queryable, cb) + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(cb) """ - @spec record_batches(queryable(), batch_options(), record_batch_callback()) :: [] - def record_batches(queryable, opts \\ [], callback) do - query_batches(queryable, opts, &callback.(Repo.all(&1))) + @spec record_batches(queryable(), batch_options()) :: Enumerable.t() + def record_batches(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.map(&Repo.all/1) end @doc """ - Execute a callback with bulk queries on a queryable, using batches to avoid locking. + Stream bulk queries on a queryable, using batches to avoid locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -76,41 +88,36 @@ defmodule PhilomenaQuery.Batch do > If you are looking to receive schema structures (e.g., you are querying for `Image`s, > and you want to receive `Image` objects, then use `record_batches/3` instead. - An `m:Ecto.Query` which selects all IDs in the current batch is passed into the callback - during each invocation. + `m:Ecto.Query` structs which select the IDs in each batch are streamed out. ## Example queryable = from ui in ImageVote, where: ui.user_id == 1234 - opts = [id_field: :image_id] - - cb = fn bulk_query -> - Repo.delete_all(bulk_query) - end - - PhilomenaQuery.Batch.query_batches(queryable, opts, cb) + queryable + |> PhilomenaQuery.Batch.query_batches(id_field: :image_id) + |> Enum.each(fn batch_query -> Repo.delete_all(batch_query) end) """ - @spec query_batches(queryable(), batch_options(), query_batch_callback()) :: [] - def query_batches(queryable, opts \\ [], callback) do - ids = load_ids(queryable, -1, opts) - - query_batches(queryable, opts, callback, ids) - end - - defp query_batches(_queryable, _opts, _callback, []), do: [] - - defp query_batches(queryable, opts, callback, ids) do + @spec query_batches(queryable(), batch_options()) :: Enumerable.t(Ecto.Query.t()) + def query_batches(queryable, opts \\ []) do id_field = Keyword.get(opts, :id_field, :id) - queryable - |> where([m], field(m, ^id_field) in ^ids) - |> callback.() + Stream.unfold( + load_ids(queryable, -1, opts), + fn + [] -> + # Stop when no more results are produced + nil - ids = load_ids(queryable, Enum.max(ids), opts) + ids -> + # Process results and output next query + output = where(queryable, [m], field(m, ^id_field) in ^ids) + next_ids = load_ids(queryable, Enum.max(ids), opts) - query_batches(queryable, opts, callback, ids) + {output, next_ids} + end + ) end defp load_ids(queryable, max_id, opts) do @@ -118,8 +125,9 @@ defmodule PhilomenaQuery.Batch do batch_size = Keyword.get(opts, :batch_size, 1000) queryable - |> exclude(:preload) |> exclude(:order_by) + |> exclude(:preload) + |> exclude(:select) |> order_by(asc: ^id_field) |> where([m], field(m, ^id_field) > ^max_id) |> select([m], field(m, ^id_field)) diff --git a/lib/philomena_query/search.ex b/lib/philomena_query/search.ex index 2519e580..cd02137c 100644 --- a/lib/philomena_query/search.ex +++ b/lib/philomena_query/search.ex @@ -199,11 +199,13 @@ defmodule PhilomenaQuery.Search do Search.reindex(query, Image, batch_size: 5000) """ - @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: [] + @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok def reindex(queryable, module, opts \\ []) do index = @policy.index_for(module) - Batch.record_batches(queryable, opts, fn records -> + queryable + |> Batch.record_batches(opts) + |> Enum.each(fn records -> lines = Enum.flat_map(records, fn record -> doc = index.as_json(record) diff --git a/lib/philomena_web/controllers/image/source_controller.ex b/lib/philomena_web/controllers/image/source_controller.ex index 373dcc10..2dbc4f47 100644 --- a/lib/philomena_web/controllers/image/source_controller.ex +++ b/lib/philomena_web/controllers/image/source_controller.ex @@ -4,6 +4,7 @@ defmodule PhilomenaWeb.Image.SourceController do alias Philomena.SourceChanges.SourceChange alias Philomena.UserStatistics alias Philomena.Images.Image + alias Philomena.Images.Source alias Philomena.Images alias Philomena.Repo import Ecto.Query @@ -41,7 +42,9 @@ defmodule PhilomenaWeb.Image.SourceController do PhilomenaWeb.Api.Json.ImageView.render("show.json", %{image: image, interactions: []}) ) - changeset = Images.change_image(image) + changeset = + %{image | sources: sources_for_edit(image.sources)} + |> Images.change_image() source_change_count = SourceChange @@ -74,4 +77,9 @@ defmodule PhilomenaWeb.Image.SourceController do ) end end + + # TODO: this is duplicated in ImageController + defp sources_for_edit(), do: [%Source{}] + defp sources_for_edit([]), do: sources_for_edit() + defp sources_for_edit(sources), do: sources end diff --git a/lib/philomena_web/controllers/image_controller.ex b/lib/philomena_web/controllers/image_controller.ex index c66c2fe7..2eeb3c5a 100644 --- a/lib/philomena_web/controllers/image_controller.ex +++ b/lib/philomena_web/controllers/image_controller.ex @@ -219,6 +219,7 @@ defmodule PhilomenaWeb.ImageController do end end + # TODO: this is duplicated in Image.SourceController defp sources_for_edit(), do: [%Source{}] defp sources_for_edit([]), do: sources_for_edit() defp sources_for_edit(sources), do: sources diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex index 7caff5c9..0b108008 100644 --- a/lib/philomena_web/markdown_renderer.ex +++ b/lib/philomena_web/markdown_renderer.ex @@ -75,8 +75,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do defp render_representations(images, conn) do loaded_images = load_images(images) - images - |> Enum.map(fn group -> + Map.new(images, fn group -> img = loaded_images[Enum.at(group, 0)] text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}" @@ -131,8 +130,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do |> Phoenix.HTML.Safe.to_iodata() |> IO.iodata_to_binary() - [text, string_contents] + {text, string_contents} end) - |> Map.new(fn [id, html] -> {id, html} end) end end diff --git a/lib/philomena_web/templates/commission/_directory_results.html.slime b/lib/philomena_web/templates/commission/_directory_results.html.slime index 8071b52f..47883dc7 100644 --- a/lib/philomena_web/templates/commission/_directory_results.html.slime +++ b/lib/philomena_web/templates/commission/_directory_results.html.slime @@ -62,3 +62,6 @@ elixir: - true -> p We couldn't find any commission listings to display. Sorry! + + .block__header.page__header + .page__pagination = pagination diff --git a/lib/philomena_web/templates/search/_form.html.slime b/lib/philomena_web/templates/search/_form.html.slime index d1ceb34d..8ffeef48 100644 --- a/lib/philomena_web/templates/search/_form.html.slime +++ b/lib/philomena_web/templates/search/_form.html.slime @@ -1,7 +1,7 @@ h1 Search = form_for :search, ~p"/search", [id: "searchform", method: "get", class: "js-search-form", enforce_utf8: false], fn f -> - = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"] + = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"], data: [ac: "true", ac_min_length: 3, ac_mode: "search"] .block .block__header.flex diff --git a/mix.exs b/mix.exs index 4032d859..1dde7ea0 100644 --- a/mix.exs +++ b/mix.exs @@ -77,7 +77,7 @@ defmodule Philomena.MixProject do {:mail, "~> 0.3.0"}, # Markdown - {:rustler, "~> 0.27"}, + {:rustler, "~> 0.35"}, # Linting {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 7f179ae0..35d487fe 100644 --- a/mix.lock +++ b/mix.lock @@ -5,14 +5,14 @@ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "canada": {:hex, :canada, "1.0.2", "040e4c47609b0a67d5773ac1fbe5e99f840cef173d69b739beda7c98453e0770", [:mix], [], "hexpm", "4269f74153fe89583fe50bd4d5de57bfe01f31258a6b676d296f3681f1483c68"}, "canary": {:git, "https://github.com/marcinkoziej/canary.git", "704debde7a2c0600f78c687807884bf37c45bd79", [ref: "704debde7a2c0600f78c687807884bf37c45bd79"]}, - "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "credo_envvar": {:hex, :credo_envvar, "0.1.4", "40817c10334e400f031012c0510bfa0d8725c19d867e4ae39cf14f2cbebc3b20", [:mix], [{:credo, "~> 1.0", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "5055cdb4bcbaf7d423bc2bb3ac62b4e2d825e2b1e816884c468dee59d0363009"}, "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, "ecto": {:hex, :ecto, "3.12.1", "626765f7066589de6fa09e0876a253ff60c3d00870dd3a1cd696e2ba67bfceea", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df0045ab9d87be947228e05a8d153f3e06e0d05ab10c3b3cc557d2f7243d1940"}, @@ -27,7 +27,7 @@ "expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"}, "exq": {:hex, :exq, "0.19.0", "06eb92944dad39f0954dc8f63190d3e24d11734eef88cf5800883e57ebf74f3c", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 6.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "24fc0ebdd87cc7406e1034fb46c2419f9c8a362f0ec634d23b6b819514d36390"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, "gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, @@ -62,8 +62,8 @@ "qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"}, "redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, - "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"}, - "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"}, + "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"}, + "rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]}, "secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"}, @@ -71,7 +71,7 @@ "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"}, "swoosh": {:hex, :swoosh, "1.16.10", "04be6e2eb1a31aa0aa21a731175c81cc3998189456a92daf13d44a5c754afcf5", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "756be04db173c0cbe318f1dfe2bcc88aa63aed78cf5a4b02b61b36ee11fc716a"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 0ca74719..54aa5826 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -11,12 +17,46 @@ dependencies = [ "memchr", ] +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "base64" version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "bon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -29,6 +69,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "caseless" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" +dependencies = [ + "regex", + "unicode-normalization", +] + [[package]] name = "cc" version = "1.0.94" @@ -43,12 +93,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" -version = "0.24.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a" +version = "0.29.0" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.2#00ac2a12d5797feb0ceba9a98487451ab65593fe" dependencies = [ - "derive_builder", + "bon", + "caseless", "entities", - "http", "memchr", "once_cell", "regex", @@ -58,10 +108,25 @@ dependencies = [ ] [[package]] -name = "darling" -version = "0.20.9" +name = "crc32fast" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -69,9 +134,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", @@ -83,9 +148,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", @@ -93,48 +158,55 @@ dependencies = [ ] [[package]] -name = "derive_builder" -version = "0.20.0" +name = "derive_arbitrary" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" -dependencies = [ - "darling", "proc-macro2", "quote", "syn", ] -[[package]] -name = "derive_builder_macro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" -dependencies = [ - "derive_builder_core", - "syn", -] - [[package]] name = "deunicode" version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322ef0094744e63628e6f0eb2295517f79276a5b342a4c2ff3042566ca181d4e" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "entities" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -151,10 +223,16 @@ dependencies = [ ] [[package]] -name = "heck" -version = "0.4.1" +name = "hashbrown" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "http" @@ -183,6 +261,22 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + [[package]] name = "itoa" version = "1.0.11" @@ -218,18 +312,28 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -238,15 +342,24 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "percent-encoding" @@ -260,17 +373,31 @@ version = "0.3.0" dependencies = [ "base64", "comrak", + "http", "jemallocator", + "once_cell", + "regex", "ring", "rustler", "url", + "zip", +] + +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -307,6 +434,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.3" @@ -330,36 +463,40 @@ dependencies = [ [[package]] name = "rustler" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7a2f98cb272ae0548e434bb3afde626012084cbebef84542bc7afed868bd6f" +checksum = "b705f2c3643cc170d8888cb6bad589155d9c0248f3104ef7a04c2b7ffbaf13fc" dependencies = [ - "lazy_static", + "inventory", + "libloading", + "regex-lite", "rustler_codegen", - "rustler_sys", ] [[package]] name = "rustler_codegen" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae37fb154683a1ff13e95dfd16c15043d6aee9853ea650ca34a45bd94e319a5" +checksum = "3ad56caff00562948bd6ac33c18dbc579e5a1bbee2d7f2f54073307e57f6b57a" dependencies = [ "heck", + "inventory", "proc-macro2", "quote", "syn", ] [[package]] -name = "rustler_sys" -version = "2.2.1" +name = "rustversion" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0eb19e2fdf4abc3662441087fc499a58b7550822cc8007e79f15c40157c883" -dependencies = [ - "regex", - "unreachable", -] +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "slug" @@ -385,15 +522,35 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.59" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -442,15 +599,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -468,12 +616,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -559,3 +701,98 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 942d39bc..e6bba775 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -10,12 +10,16 @@ path = "src/lib.rs" crate-type = ["dylib"] [dependencies] -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } -jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } -rustler = "0.28" -ring = "0.16" base64 = "0.21" +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false } +http = "0.2" +jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } +once_cell = "1.20" +regex = "1" +ring = "0.16" +rustler = "0.35" url = "2.3" +zip = { version = "2.2.0", features = ["deflate"], default-features = false } [profile.release] opt-level = 3 diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index 5a72b3f6..c79f12e1 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -1,8 +1,8 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; use ring::hmac; use std::env; use url::Url; -use base64::Engine; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; fn trusted_host(mut url: Url) -> Option { url.set_port(Some(443)).ok()?; @@ -11,7 +11,7 @@ fn trusted_host(mut url: Url) -> Option { Some(url.to_string()) } -fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option { +fn untrusted_host(url: Url, camo_host: &str, camo_key: &str) -> Option { let camo_url = format!("https://{}", camo_host); let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref()); let tag = hmac::sign(&key, url.to_string().as_bytes()); @@ -27,20 +27,24 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option Option { +pub fn image_url(uri: &str) -> Option { let cdn_host = env::var("CDN_HOST").ok()?; - let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| String::from("")); - let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| String::from("")); + let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| "".into()); + let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| "".into()); if camo_key.is_empty() { - return Some(uri); + return Some(uri.into()); } - let url = Url::parse(&uri).ok()?; + let url = Url::parse(uri).ok()?; match url.host_str() { Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url), - Some(_) => untrusted_host(url, camo_host, camo_key), - None => Some(String::from("")), + Some(_) => untrusted_host(url, &camo_host, &camo_key), + None => Some("".into()), } } + +pub fn image_url_careful(uri: &str) -> String { + image_url(uri).unwrap_or_else(|| "".into()) +} diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs new file mode 100644 index 00000000..9bcacfc9 --- /dev/null +++ b/native/philomena/src/domains.rs @@ -0,0 +1,38 @@ +use std::collections::BTreeSet; +use std::env; + +use http::Uri; +use once_cell::sync::Lazy; +use regex::Regex; + +pub type DomainSet = BTreeSet; + +static DOMAINS: Lazy> = Lazy::new(|| { + if let Ok(domains) = env::var("SITE_DOMAINS") { + return Some(domains.split(',').map(|s| s.to_string()).collect()); + } + + None +}); + +pub fn get() -> &'static Option { + &DOMAINS +} + +pub fn relativize(domains: &DomainSet, url: &str) -> Option { + let uri = url.parse::().ok()?; + + if let Some(a) = uri.authority() { + if domains.contains(a.host()) { + if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) { + return Some(re.replace(url, "").into()); + } + } + } + + Some(url.into()) +} + +pub fn relativize_careful(domains: &DomainSet, url: &str) -> String { + relativize(domains, url).unwrap_or_else(|| url.into()) +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index f5317aa3..6f7f72b2 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,32 +1,58 @@ use jemallocator::Jemalloc; -use rustler::Term; +use rustler::{Atom, Binary}; +use std::collections::HashMap; mod camo; +mod domains; mod markdown; +#[cfg(test)] +mod tests; +mod zip; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; rustler::init! { - "Elixir.Philomena.Native", - [markdown_to_html, markdown_to_html_unsafe, camo_image_url] + "Elixir.Philomena.Native" } // Markdown NIF wrappers. #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html(input: String, reps: Term) -> String { +fn markdown_to_html(input: &str, reps: HashMap) -> String { markdown::to_html(input, reps) } #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html_unsafe(input: String, reps: Term) -> String { +fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String { markdown::to_html_unsafe(input, reps) } // Camo NIF wrappers. #[rustler::nif] -fn camo_image_url(input: String) -> String { - camo::image_url(input).unwrap_or_else(|| String::from("")) +fn camo_image_url(input: &str) -> String { + camo::image_url_careful(input) +} + +// Zip NIF wrappers. + +#[rustler::nif] +fn zip_open_writer(path: &str) -> Result { + zip::open_writer(path) +} + +#[rustler::nif] +fn zip_start_file(writer: zip::WriterResourceArc, name: &str) -> Atom { + zip::start_file(writer, name) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_write(writer: zip::WriterResourceArc, data: Binary) -> Atom { + zip::write(writer, data.as_slice()) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_finish(writer: zip::WriterResourceArc) -> Atom { + zip::finish(writer) } diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index af976233..fbd846ac 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,53 +1,54 @@ -use comrak::ComrakOptions; -use crate::camo; -use rustler::{MapIterator, Term}; +use crate::{camo, domains}; +use comrak::Options; use std::collections::HashMap; -use std::env; +use std::sync::Arc; -fn common_options() -> ComrakOptions { - let mut options = ComrakOptions::default(); +pub fn common_options() -> Options { + let mut options = Options::default(); + + // Upstream options options.extension.autolink = true; options.extension.table = true; options.extension.description_lists = true; options.extension.superscript = true; options.extension.strikethrough = true; - options.extension.philomena = true; options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; + options.render.escape = true; - options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); + // Philomena options + options.extension.underline = true; + options.extension.spoiler = true; + options.extension.greentext = true; + options.extension.subscript = true; + options.extension.philomena = true; + options.render.ignore_empty_links = true; + options.render.ignore_setext = true; - if let Ok(domains) = env::var("SITE_DOMAINS") { - options.extension.philomena_domains = Some(domains.split(',').map(|s| s.to_string()).collect::>()); + options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url))); + + if let Some(domains) = domains::get() { + options.extension.link_url_rewriter = Some(Arc::new(|url: &str| { + domains::relativize_careful(domains, url) + })); } options } -fn map_to_hashmap(map: Term) -> Option> { - Some(MapIterator::new(map)?.map(|(key, value)| { - let key: String = key.decode().unwrap_or_else(|_| String::from("")); - let value: String = value.decode().unwrap_or_else(|_| String::from("")); +pub fn to_html(input: &str, reps: HashMap) -> String { + let mut options = common_options(); + options.extension.replacements = Some(reps); - (key, value) - }).collect()) + comrak::markdown_to_html(input, &options) } -pub fn to_html(input: String, reps: Term) -> String { - let mut options = common_options(); - options.render.escape = true; - - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) -} - -pub fn to_html_unsafe(input: String, reps: Term) -> String { +pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { let mut options = common_options(); + options.render.escape = false; options.render.unsafe_ = true; + options.extension.replacements = Some(reps); - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) + comrak::markdown_to_html(input, &options) } diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs new file mode 100644 index 00000000..547f0f86 --- /dev/null +++ b/native/philomena/src/tests.rs @@ -0,0 +1,273 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::{domains, markdown::*}; + +fn test_options() -> comrak::Options { + let mut options = common_options(); + options.extension.image_url_rewriter = None; + options.extension.link_url_rewriter = None; + options +} + +fn html(input: &str, expected: &str) { + html_opts_w(input, expected, &test_options()); +} + +fn html_opts_i(input: &str, expected: &str, opts: F) +where + F: Fn(&mut comrak::Options), +{ + let mut options = test_options(); + opts(&mut options); + + html_opts_w(input, expected, &options); +} + +fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) { + let output = comrak::markdown_to_html(input, options); + + if output != expected { + println!("Input:"); + println!("========================"); + println!("{}", input); + println!("========================"); + println!("Expected:"); + println!("========================"); + println!("{}", expected); + println!("========================"); + println!("Output:"); + println!("========================"); + println!("{}", output); + println!("========================"); + } + assert_eq!(output, expected); +} + +#[test] +fn subscript() { + html("H~2~O\n", "
H2O
\n"); +} + +#[test] +fn subscript_autolink_interaction() { + html( + "https://example.com/search?q=1%2C2%2C3", + "\n" + ); +} + +#[test] +fn underscore_autolink_interaction() { + html( + "https://example.com/x_", + "\n" + ) +} + +#[test] +fn spoiler() { + html( + "The ||dog dies at the end of Marley and Me||.\n", + "
The dog dies at the end of Marley and Me.
\n", + ); +} + +#[test] +fn spoiler_in_table() { + html( + "Text | Result\n--- | ---\n`||some clever text||` | ||some clever text||\n", + concat!( + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
TextResult
||some clever text||some clever text
\n" + ), + ); +} + +#[test] +fn spoiler_regressions() { + html( + "|should not be spoiler|\n||should be spoiler||\n|||should be spoiler surrounded by pipes|||", + concat!( + "
|should not be spoiler|
\n", + "should be spoiler
\n", + "|should be spoiler surrounded by pipes|
\n" + ), + ); +} + +#[test] +fn mismatched_spoilers() { + html( + "|||this is a spoiler with pipe in front||\n||this is not a spoiler|\n||this is a spoiler with pipe after|||", + concat!( + "
|this is a spoiler with pipe in front
\n", + "||this is not a spoiler|
\n", + "this is a spoiler with pipe after|
\n" + ), + ); +} + +#[test] +fn underline() { + html( + "__underlined__\n", + "
underlined
\n", + ); +} + +#[test] +fn no_setext_headings_in_philomena() { + html( + "text text\n---", + "
text text
\n
\n", + ); +} + +#[test] +fn greentext_preserved() { + html( + ">implying\n>>implying", + "
>implying
\n>>implying
\n", + ); +} + +#[test] +fn separate_quotes_on_line_end() { + html( + "> 1\n>\n> 2", + "
\n
1
\n
\n
>
\n
\n
2
\n
\n" + ); +} + +#[test] +fn unnest_quotes_on_line_end() { + html( + "> 1\n> > 2\n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn unnest_quotes_on_line_end_commonmark() { + html( + "> 1\n> > 2\n> \n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn philomena_images() { + html( + "![full](http://example.com/image.png)", + "
\"full\"
\n", + ); +} + +#[test] +fn no_empty_link() { + html_opts_i( + "[](https://example.com/evil.domain.for.seo.spam)", + "
[](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); + + html_opts_i( + "[ ](https://example.com/evil.domain.for.seo.spam)", + "
[ ](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); +} + +#[test] +fn empty_image_allowed() { + html( + "![ ](https://example.com/evil.domain.for.seo.spam)", + "
\"
\n", + ); +} + +#[test] +fn image_inside_link_allowed() { + html( + "[![](https://example.com/image.png)](https://example.com/)", + "
\"\"
\n", + ); +} + +#[test] +fn image_mention() { + html_opts_i( + "hello world >>1234p >>1337", + "
hello world
p
>>1337
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn image_mention_line_start() { + html_opts_i( + ">>1234p", + "
p
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn auto_relative_links() { + let domains = Arc::new(vec!["example.com".into()].into_iter().collect()); + let f = Arc::new(move |url: &str| domains::relativize_careful(&*domains, url)); + + html_opts_i( + "[some link text](https://example.com/some/path)", + "\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path", + "\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "[some link text](https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345)", + "\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345", + "\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); +} diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs new file mode 100644 index 00000000..14489ee2 --- /dev/null +++ b/native/philomena/src/zip.rs @@ -0,0 +1,67 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::sync::Mutex; + +use rustler::{Atom, Resource, ResourceArc}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + +mod atoms { + rustler::atoms! { + ok, + error, + } +} + +pub struct WriterResource { + inner: Mutex>>, +} + +#[rustler::resource_impl] +impl Resource for WriterResource {} + +pub type WriterResourceArc = ResourceArc; + +fn with_writer(writer: WriterResourceArc, f: F) -> Atom +where + F: FnOnce(&mut Option>) -> Option, +{ + let mut guard = match writer.inner.lock() { + Ok(g) => g, + Err(_) => return atoms::error(), + }; + + match f(&mut guard) { + Some(_) => atoms::ok(), + None => atoms::error(), + } +} + +pub fn open_writer(path: &str) -> Result { + match OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(path) + { + Ok(file) => Ok(ResourceArc::new(WriterResource { + inner: Mutex::new(Some(ZipWriter::new(file))), + })), + Err(_) => Err(atoms::error()), + } +} + +pub fn start_file(writer: WriterResourceArc, name: &str) -> Atom { + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + + with_writer(writer, |writer| { + writer.as_mut()?.start_file(name, options).ok() + }) +} + +pub fn write(writer: WriterResourceArc, data: &[u8]) -> Atom { + with_writer(writer, |writer| writer.as_mut()?.write(data).ok()) +} + +pub fn finish(writer: WriterResourceArc) -> Atom { + with_writer(writer, |writer| writer.take().map(|w| w.finish().ok())) +}