From 74daa662aca37616df47ac10eac87f69e39b3e3a Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Sun, 21 Apr 2024 20:44:08 -0400
Subject: [PATCH 1/8] Fix images fast indexer
---
index/images.mk | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/index/images.mk b/index/images.mk
index c1f27a49..cc93f9c8 100644
--- a/index/images.mk
+++ b/index/images.mk
@@ -7,7 +7,7 @@ all: import_es
import_es: dump_jsonl
$(ELASTICDUMP) --input=images.jsonl --output=http://localhost:9200/ --output-index=images --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id"
-dump_jsonl: metadata true_uploaders uploaders deleters galleries tags hides upvotes downvotes faves tag_names
+dump_jsonl: metadata true_uploaders uploaders deleters galleries tags sources hides upvotes downvotes faves tag_names
psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_images.jsonb_object_agg(object) from temp_images.image_search_json group by image_id) to stdout;' > images.jsonl
psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'drop schema temp_images cascade;'
sed -i images.jsonl -e 's/\\\\/\\/g'
@@ -15,6 +15,8 @@ dump_jsonl: metadata true_uploaders uploaders deleters galleries tags hides upvo
metadata: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
insert into temp_images.image_search_json (image_id, object) select id, jsonb_build_object(
+ 'approved', approved,
+ 'animated', is_animated,
'anonymous', anonymous,
'aspect_ratio', nullif(image_aspect_ratio, 'NaN'::float8),
'comment_count', comments_count,
@@ -23,6 +25,7 @@ metadata: image_search_json
'description', description,
'downvotes', downvotes_count,
'duplicate_id', duplicate_id,
+ 'duration', (case when is_animated then image_duration else 0::float end),
'faves', faves_count,
'file_name', image_name,
'fingerprint', fingerprint,
@@ -35,10 +38,11 @@ metadata: image_search_json
'orig_sha512_hash', image_orig_sha512_hash,
'original_format', image_format,
'pixels', cast(image_width as bigint)*cast(image_height as bigint),
+ 'processed', processed,
'score', score,
'size', image_size,
'sha512_hash', image_sha512_hash,
- 'source_url', lower(source_url),
+ 'thumbnails_generated', thumbnails_generated,
'updated_at', updated_at,
'upvotes', upvotes_count,
'width', image_width,
@@ -64,8 +68,6 @@ deleters: image_search_json
galleries: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
insert into temp_images.image_search_json (image_id, object) select gi.image_id, jsonb_build_object('gallery_interactions', jsonb_agg(jsonb_build_object('id', gi.gallery_id, 'position', gi.position))) from gallery_interactions gi group by image_id;
- insert into temp_images.image_search_json (image_id, object) select gi.image_id, jsonb_build_object('gallery_id', jsonb_agg(gi.gallery_id)) from gallery_interactions gi group by image_id;
- insert into temp_images.image_search_json (image_id, object) select gi.image_id, jsonb_build_object('gallery_position', jsonb_object_agg(gi.gallery_id, gi.position)) from gallery_interactions gi group by image_id;
SQL
tags: image_search_json
@@ -73,24 +75,29 @@ tags: image_search_json
insert into temp_images.image_search_json (image_id, object) select it.image_id, jsonb_build_object('tag_ids', jsonb_agg(it.tag_id), 'tag_count', count(*)) from image_taggings it group by image_id;
SQL
+sources: image_search_json
+ psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
+ insert into temp_images.image_search_json (image_id, object) select s.image_id, jsonb_build_object('source_url', jsonb_agg(lower(s.source)), 'source_count', count(*)) from image_sources s group by image_id;
+ SQL
+
hides: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
- insert into temp_images.image_search_json (image_id, object) select ih.image_id, jsonb_build_object('hidden_by_ids', jsonb_agg(ih.user_id), 'hidden_by', jsonb_agg(lower(u.name))) from image_hides ih inner join users u on u.id = ih.user_id group by image_id;
+ insert into temp_images.image_search_json (image_id, object) select ih.image_id, jsonb_build_object('hidden_by_user_ids', jsonb_agg(ih.user_id), 'hidden_by_users', jsonb_agg(lower(u.name))) from image_hides ih inner join users u on u.id = ih.user_id group by image_id;
SQL
downvotes: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
- insert into temp_images.image_search_json (image_id, object) select iv.image_id, jsonb_build_object('downvoted_by_ids', jsonb_agg(iv.user_id), 'downvoted_by', jsonb_agg(lower(u.name))) from image_votes iv inner join users u on u.id = iv.user_id where iv.up = false group by image_id;
+ insert into temp_images.image_search_json (image_id, object) select iv.image_id, jsonb_build_object('downvoter_ids', jsonb_agg(iv.user_id), 'downvoters', jsonb_agg(lower(u.name))) from image_votes iv inner join users u on u.id = iv.user_id where iv.up = false group by image_id;
SQL
upvotes: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
- insert into temp_images.image_search_json (image_id, object) select iv.image_id, jsonb_build_object('upvoted_by_ids', jsonb_agg(iv.user_id), 'upvoted_by', jsonb_agg(lower(u.name))) from image_votes iv inner join users u on u.id = iv.user_id where iv.up = true group by image_id;
+ insert into temp_images.image_search_json (image_id, object) select iv.image_id, jsonb_build_object('upvoter_ids', jsonb_agg(iv.user_id), 'upvoters', jsonb_agg(lower(u.name))) from image_votes iv inner join users u on u.id = iv.user_id where iv.up = true group by image_id;
SQL
faves: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
- insert into temp_images.image_search_json (image_id, object) select if.image_id, jsonb_build_object('faved_by_ids', jsonb_agg(if.user_id), 'faved_by', jsonb_agg(lower(u.name))) from image_faves if inner join users u on u.id = if.user_id group by image_id;
+ insert into temp_images.image_search_json (image_id, object) select if.image_id, jsonb_build_object('favourited_by_user_ids', jsonb_agg(if.user_id), 'favourited_by_users', jsonb_agg(lower(u.name))) from image_faves if inner join users u on u.id = if.user_id group by image_id;
SQL
tag_names: tags_with_aliases
From f9a6240014ff76339f5ee8c4dfaa8458cac9d97e Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Sun, 21 Apr 2024 21:20:43 -0400
Subject: [PATCH 2/8] Add tag category counts to index
---
index/images.mk | 15 ++++++++++++-
lib/philomena/images/elasticsearch_index.ex | 24 +++++++++++++++++++--
2 files changed, 36 insertions(+), 3 deletions(-)
diff --git a/index/images.mk b/index/images.mk
index cc93f9c8..8c843ee2 100644
--- a/index/images.mk
+++ b/index/images.mk
@@ -72,7 +72,20 @@ galleries: image_search_json
tags: image_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
- insert into temp_images.image_search_json (image_id, object) select it.image_id, jsonb_build_object('tag_ids', jsonb_agg(it.tag_id), 'tag_count', count(*)) from image_taggings it group by image_id;
+ insert into temp_images.image_search_json (image_id, object) select it.image_id, jsonb_build_object(
+ 'tag_ids', jsonb_agg(it.tag_id),
+ 'tag_count', count(*),
+ 'error_tag_count', count(case when t.category = 'error' then t.category else null end),
+ 'rating_tag_count', count(case when t.category = 'rating' then t.category else null end),
+ 'origin_tag_count', count(case when t.category = 'origin' then t.category else null end),
+ 'character_tag_count', count(case when t.category = 'character' then t.category else null end),
+ 'oc_tag_count', count(case when t.category = 'oc' then t.category else null end),
+ 'species_tag_count', count(case when t.category = 'species' then t.category else null end),
+ 'body_type_tag_count', count(case when t.category = 'body-type' then t.category else null end),
+ 'content_fanmade_tag_count', count(case when t.category = 'content-fanmade' then t.category else null end),
+ 'content_official_tag_count', count(case when t.category = 'content-official' then t.category else null end),
+ 'spoiler_tag_count', count(case when t.category = 'spoiler' then t.category else null end),
+ ) from image_taggings it inner join tags t on t.id = it.tag_id group by image_id;
SQL
sources: image_search_json
diff --git a/lib/philomena/images/elasticsearch_index.ex b/lib/philomena/images/elasticsearch_index.ex
index b76912bd..64872d1a 100644
--- a/lib/philomena/images/elasticsearch_index.ex
+++ b/lib/philomena/images/elasticsearch_index.ex
@@ -87,7 +87,17 @@ defmodule Philomena.Images.ElasticsearchIndex do
namespace: %{type: "keyword"}
}
},
- approved: %{type: "boolean"}
+ approved: %{type: "boolean"},
+ error_tag_count: %{type: "integer"},
+ rating_tag_count: %{type: "integer"},
+ origin_tag_count: %{type: "integer"},
+ character_tag_count: %{type: "integer"},
+ oc_tag_count: %{type: "integer"},
+ species_tag_count: %{type: "integer"},
+ body_type_tag_count: %{type: "integer"},
+ content_fanmade_tag_count: %{type: "integer"},
+ content_official_tag_count: %{type: "integer"},
+ spoiler_tag_count: %{type: "integer"}
}
}
}
@@ -151,7 +161,17 @@ defmodule Philomena.Images.ElasticsearchIndex do
upvoters: image.upvoters |> Enum.map(&String.downcase(&1.name)),
downvoters: image.downvoters |> Enum.map(&String.downcase(&1.name)),
deleted_by_user: if(!!image.deleter, do: image.deleter.name),
- approved: image.approved
+ approved: image.approved,
+ error_tag_count: Enum.count(image.tags, &(&1.category == "error")),
+ rating_tag_count: Enum.count(image.tags, &(&1.category == "rating")),
+ origin_tag_count: Enum.count(image.tags, &(&1.category == "origin")),
+ character_tag_count: Enum.count(image.tags, &(&1.category == "character")),
+ oc_tag_count: Enum.count(image.tags, &(&1.category == "oc")),
+ species_tag_count: Enum.count(image.tags, &(&1.category == "species")),
+ body_type_tag_count: Enum.count(image.tags, &(&1.category == "body-type")),
+ content_fanmade_tag_count: Enum.count(image.tags, &(&1.category == "content-fanmade")),
+ content_official_tag_count: Enum.count(image.tags, &(&1.category == "content-official")),
+ spoiler_tag_count: Enum.count(image.tags, &(&1.category == "spoiler"))
}
end
From ea25f2a01e3250ebde83901ba26375fc36cd19a6 Mon Sep 17 00:00:00 2001
From: mdashlw <mdashlw@gmail.com>
Date: Mon, 22 Apr 2024 04:21:00 +0300
Subject: [PATCH 3/8] Source count index (#214)
* elasticsearch image index: add source_count
* images query: add source_count field
---
lib/philomena/images/elasticsearch_index.ex | 2 ++
lib/philomena/images/query.ex | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/philomena/images/elasticsearch_index.ex b/lib/philomena/images/elasticsearch_index.ex
index 64872d1a..0e5fb296 100644
--- a/lib/philomena/images/elasticsearch_index.ex
+++ b/lib/philomena/images/elasticsearch_index.ex
@@ -56,6 +56,7 @@ defmodule Philomena.Images.ElasticsearchIndex do
size: %{type: "integer"},
sha512_hash: %{type: "keyword"},
source_url: %{type: "keyword"},
+ source_count: %{type: "integer"},
tag_count: %{type: "integer"},
tag_ids: %{type: "keyword"},
tags: %{type: "text", analyzer: "keyword"},
@@ -130,6 +131,7 @@ defmodule Philomena.Images.ElasticsearchIndex do
uploader: if(!!image.user and !image.anonymous, do: String.downcase(image.user.name)),
true_uploader: if(!!image.user, do: String.downcase(image.user.name)),
source_url: image.sources |> Enum.map(&String.downcase(&1.source)),
+ source_count: length(image.sources),
file_name: image.image_name,
original_format: image.image_format,
processed: image.processed,
diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex
index 81551580..69effbbd 100644
--- a/lib/philomena/images/query.ex
+++ b/lib/philomena/images/query.ex
@@ -69,7 +69,7 @@ defmodule Philomena.Images.Query do
defp anonymous_fields do
[
int_fields:
- ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size),
+ ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size source_count),
float_fields: ~W(aspect_ratio wilson_score duration),
date_fields: ~W(created_at updated_at first_seen_at),
literal_fields:
From 3f1f208916aa5c0bb62e336d81381ac1fdf9f1e0 Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Sun, 21 Apr 2024 21:22:02 -0400
Subject: [PATCH 4/8] Drop source_count from fields pending reindex
---
lib/philomena/images/query.ex | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex
index 69effbbd..81551580 100644
--- a/lib/philomena/images/query.ex
+++ b/lib/philomena/images/query.ex
@@ -69,7 +69,7 @@ defmodule Philomena.Images.Query do
defp anonymous_fields do
[
int_fields:
- ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size source_count),
+ ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size),
float_fields: ~W(aspect_ratio wilson_score duration),
date_fields: ~W(created_at updated_at first_seen_at),
literal_fields:
From ac3b15b1e2edf8852eba53623dc230a19b65dcb6 Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Mon, 22 Apr 2024 08:29:38 -0400
Subject: [PATCH 5/8] Add tag count fields
---
lib/philomena/images/query.ex | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex
index 81551580..db00a20b 100644
--- a/lib/philomena/images/query.ex
+++ b/lib/philomena/images/query.ex
@@ -66,10 +66,26 @@ defmodule Philomena.Images.Query do
end
end
+ defp tag_count_fields do
+ [
+ "body_type_tag_count",
+ "error_tag_count",
+ "character_tag_count",
+ "content_fanmade_tag_count",
+ "content_official_tag_count",
+ "oc_tag_count",
+ "origin_tag_count",
+ "rating_tag_count",
+ "species_tag_count",
+ "spoiler_tag_count"
+ ]
+ end
+
defp anonymous_fields do
[
int_fields:
- ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size),
+ ~W(id width height score upvotes downvotes faves uploader_id faved_by_id pixels size comment_count source_count tag_count) ++
+ tag_count_fields(),
float_fields: ~W(aspect_ratio wilson_score duration),
date_fields: ~W(created_at updated_at first_seen_at),
literal_fields:
From 88a1131f35518dc52135487341b6868a45e1a1f7 Mon Sep 17 00:00:00 2001
From: liamwhite <liamwhite@users.noreply.github.com>
Date: Mon, 22 Apr 2024 18:43:27 -0400
Subject: [PATCH 6/8] input-duplicator: migrate to TypeScript (#230)
---
assets/js/__tests__/input-duplicator.spec.ts | 91 ++++++++++++++++++++
assets/js/input-duplicator.js | 83 ------------------
assets/js/input-duplicator.ts | 76 ++++++++++++++++
3 files changed, 167 insertions(+), 83 deletions(-)
create mode 100644 assets/js/__tests__/input-duplicator.spec.ts
delete mode 100644 assets/js/input-duplicator.js
create mode 100644 assets/js/input-duplicator.ts
diff --git a/assets/js/__tests__/input-duplicator.spec.ts b/assets/js/__tests__/input-duplicator.spec.ts
new file mode 100644
index 00000000..fc7adf0b
--- /dev/null
+++ b/assets/js/__tests__/input-duplicator.spec.ts
@@ -0,0 +1,91 @@
+import { inputDuplicatorCreator } from '../input-duplicator';
+import { assertNotNull } from '../utils/assert';
+import { $, $$, removeEl } from '../utils/dom';
+
+describe('Input duplicator functionality', () => {
+ beforeEach(() => {
+ document.documentElement.insertAdjacentHTML('beforeend', `<form action="/">
+ <div class="js-max-input-count">3</div>
+ <div class="js-input-source">
+ <input id="0" name="0" class="js-input" type="text"/>
+ <label>
+ <a href="#" class="js-remove-input">Delete</a>
+ </label>
+ </div>
+ <div class="js-button-container">
+ <button type="button" class="js-add-input">Add input</button>
+ </div>
+ </form>`);
+ });
+
+ afterEach(() => {
+ removeEl($$<HTMLFormElement>('form'));
+ });
+
+ function runCreator() {
+ inputDuplicatorCreator({
+ addButtonSelector: '.js-add-input',
+ fieldSelector: '.js-input-source',
+ maxInputCountSelector: '.js-max-input-count',
+ removeButtonSelector: '.js-remove-input',
+ });
+ }
+
+ it('should ignore forms without a duplicator button', () => {
+ removeEl($$<HTMLButtonElement>('button'));
+ expect(runCreator()).toBeUndefined();
+ });
+
+ it('should duplicate the input elements', () => {
+ runCreator();
+
+ expect($$('input')).toHaveLength(1);
+
+ assertNotNull($<HTMLButtonElement>('.js-add-input')).click();
+
+ expect($$('input')).toHaveLength(2);
+ });
+
+ it('should duplicate the input elements when the button is before the inputs', () => {
+ const form = assertNotNull($<HTMLFormElement>('form'));
+ const buttonDiv = assertNotNull($<HTMLDivElement>('.js-button-container'));
+ removeEl(buttonDiv);
+ form.insertAdjacentElement('afterbegin', buttonDiv);
+ runCreator();
+
+ assertNotNull($<HTMLButtonElement>('.js-add-input')).click();
+
+ expect($$('input')).toHaveLength(2);
+ });
+
+ it('should not create more input elements than the limit', () => {
+ runCreator();
+
+ for (let i = 0; i < 5; i += 1) {
+ assertNotNull($<HTMLButtonElement>('.js-add-input')).click();
+ }
+
+ expect($$('input')).toHaveLength(3);
+ });
+
+ it('should remove duplicated input elements', () => {
+ runCreator();
+
+ assertNotNull($<HTMLButtonElement>('.js-add-input')).click();
+ assertNotNull($<HTMLAnchorElement>('.js-remove-input')).click();
+
+ expect($$('input')).toHaveLength(1);
+ });
+
+ it('should not remove the last input element', () => {
+ runCreator();
+
+ assertNotNull($<HTMLAnchorElement>('.js-remove-input')).click();
+ assertNotNull($<HTMLAnchorElement>('.js-remove-input')).click();
+ for (let i = 0; i < 5; i += 1) {
+ assertNotNull($<HTMLAnchorElement>('.js-remove-input')).click();
+ }
+
+ expect($$('input')).toHaveLength(1);
+ });
+});
diff --git a/assets/js/input-duplicator.js b/assets/js/input-duplicator.js
deleted file mode 100644
index 2ffa89bc..00000000
--- a/assets/js/input-duplicator.js
+++ /dev/null
@@ -1,83 +0,0 @@
-import { $, $$, disableEl, enableEl, removeEl } from './utils/dom';
-import { delegate, leftClick } from './utils/events';
-
-/**
- * @typedef InputDuplicatorOptions
- * @property {string} addButtonSelector
- * @property {string} fieldSelector
- * @property {string} maxInputCountSelector
- * @property {string} removeButtonSelector
- */
-
-/**
- * @param {InputDuplicatorOptions} options
- */
-function inputDuplicatorCreator({
- addButtonSelector,
- fieldSelector,
- maxInputCountSelector,
- removeButtonSelector
-}) {
- const addButton = $(addButtonSelector);
- if (!addButton) {
- return;
- }
-
- const form = addButton.closest('form');
- const fieldRemover = (event, target) => {
- event.preventDefault();
-
- // Prevent removing the final field element to not "brick" the form
- const existingFields = $$(fieldSelector, form);
- if (existingFields.length <= 1) {
- return;
- }
-
- removeEl(target.closest(fieldSelector));
- enableEl(addButton);
- };
-
- delegate(document, 'click', {
- [removeButtonSelector]: leftClick(fieldRemover)
- });
-
-
- const maxOptionCount = parseInt($(maxInputCountSelector, form).innerHTML, 10);
- addButton.addEventListener('click', e => {
- e.preventDefault();
-
- const existingFields = $$(fieldSelector, form);
- let existingFieldsLength = existingFields.length;
- if (existingFieldsLength < maxOptionCount) {
- // The last element matched by the `fieldSelector` will be the last field, make a copy
- const prevField = existingFields[existingFieldsLength - 1];
- const prevFieldCopy = prevField.cloneNode(true);
- const prevFieldCopyInputs = $$('input', prevFieldCopy);
- prevFieldCopyInputs.forEach(prevFieldCopyInput => {
- // Reset new input's value
- prevFieldCopyInput.value = '';
- prevFieldCopyInput.removeAttribute('value');
- // Increment sequential attributes of the input
- ['name', 'id'].forEach(attr => {
- prevFieldCopyInput.setAttribute(attr, prevFieldCopyInput[attr].replace(/\d+/g, `${existingFieldsLength}`));
- });
- });
-
- // Insert copy before the last field's next sibling, or if none, at the end of its parent
- if (prevField.nextElementSibling) {
- prevField.parentNode.insertBefore(prevFieldCopy, prevField.nextElementSibling);
- }
- else {
- prevField.parentNode.appendChild(prevFieldCopy);
- }
- existingFieldsLength++;
- }
-
- // Remove the button if we reached the max number of options
- if (existingFieldsLength >= maxOptionCount) {
- disableEl(addButton);
- }
- });
-}
-
-export { inputDuplicatorCreator };
diff --git a/assets/js/input-duplicator.ts b/assets/js/input-duplicator.ts
new file mode 100644
index 00000000..e82c892d
--- /dev/null
+++ b/assets/js/input-duplicator.ts
@@ -0,0 +1,76 @@
+import { assertNotNull } from './utils/assert';
+import { $, $$, disableEl, enableEl, removeEl } from './utils/dom';
+import { delegate, leftClick } from './utils/events';
+
+export interface InputDuplicatorOptions {
+ addButtonSelector: string;
+ fieldSelector: string;
+ maxInputCountSelector: string;
+ removeButtonSelector: string;
+}
+
+export function inputDuplicatorCreator({
+ addButtonSelector,
+ fieldSelector,
+ maxInputCountSelector,
+ removeButtonSelector
+}: InputDuplicatorOptions) {
+ const addButton = $<HTMLButtonElement>(addButtonSelector);
+ if (!addButton) {
+ return;
+ }
+
+ const form = assertNotNull(addButton.closest('form'));
+ const fieldRemover = (event: MouseEvent, target: HTMLElement) => {
+ event.preventDefault();
+
+ // Prevent removing the final field element to not "brick" the form
+ const existingFields = $$(fieldSelector, form);
+ if (existingFields.length <= 1) {
+ return;
+ }
+
+ removeEl(assertNotNull(target.closest<HTMLElement>(fieldSelector)));
+ enableEl(addButton);
+ };
+
+ delegate(form, 'click', {
+ [removeButtonSelector]: leftClick(fieldRemover)
+ });
+
+
+ const maxOptionCountElement = assertNotNull($(maxInputCountSelector, form));
+ const maxOptionCount = parseInt(maxOptionCountElement.innerHTML, 10);
+
+ addButton.addEventListener('click', e => {
+ e.preventDefault();
+
+ const existingFields = $$<HTMLElement>(fieldSelector, form);
+ let existingFieldsLength = existingFields.length;
+
+ if (existingFieldsLength < maxOptionCount) {
+ // The last element matched by the `fieldSelector` will be the last field, make a copy
+ const prevField = existingFields[existingFieldsLength - 1];
+ const prevFieldCopy = prevField.cloneNode(true) as HTMLElement;
+
+ $$<HTMLInputElement>('input', prevFieldCopy).forEach(prevFieldCopyInput => {
+ // Reset new input's value
+ prevFieldCopyInput.value = '';
+ prevFieldCopyInput.removeAttribute('value');
+
+ // Increment sequential attributes of the input
+ prevFieldCopyInput.setAttribute('name', prevFieldCopyInput.name.replace(/\d+/g, `${existingFieldsLength}`));
+ prevFieldCopyInput.setAttribute('id', prevFieldCopyInput.id.replace(/\d+/g, `${existingFieldsLength}`));
+ });
+
+ prevField.insertAdjacentElement('afterend', prevFieldCopy);
+
+ existingFieldsLength++;
+ }
+
+ // Remove the button if we reached the max number of options
+ if (existingFieldsLength >= maxOptionCount) {
+ disableEl(addButton);
+ }
+ });
+}
From df2e336a24011bd8806ad56ffc7cbf593ae54ff0 Mon Sep 17 00:00:00 2001
From: liamwhite <liamwhite@users.noreply.github.com>
Date: Mon, 22 Apr 2024 18:43:36 -0400
Subject: [PATCH 7/8] upload: add pinning test (#231)
---
assets/fix-jsdom.ts | 13 ++
assets/jest.config.js | 2 +-
assets/js/__tests__/ujs.spec.ts | 14 +--
assets/js/__tests__/upload-test.png | Bin 0 -> 527 bytes
assets/js/__tests__/upload-test.webm | Bin 0 -> 555 bytes
assets/js/__tests__/upload.spec.ts | 178 +++++++++++++++++++++++++++
assets/js/upload.js | 12 +-
assets/test/fix-event-listeners.ts | 26 ++++
8 files changed, 225 insertions(+), 20 deletions(-)
create mode 100644 assets/fix-jsdom.ts
create mode 100644 assets/js/__tests__/upload-test.png
create mode 100644 assets/js/__tests__/upload-test.webm
create mode 100644 assets/js/__tests__/upload.spec.ts
create mode 100644 assets/test/fix-event-listeners.ts
diff --git a/assets/fix-jsdom.ts b/assets/fix-jsdom.ts
new file mode 100644
index 00000000..d83d15d2
--- /dev/null
+++ b/assets/fix-jsdom.ts
@@ -0,0 +1,13 @@
+import JSDOMEnvironment from 'jest-environment-jsdom';
+
+export default class FixJSDOMEnvironment extends JSDOMEnvironment {
+ constructor(...args: ConstructorParameters<typeof JSDOMEnvironment>) {
+ super(...args);
+
+ // https://github.com/jsdom/jsdom/issues/1721#issuecomment-1484202038
+ // jsdom URL and Blob are missing most of the implementation
+ // Use the node version of these types instead
+ this.global.URL = URL;
+ this.global.Blob = Blob;
+ }
+}
diff --git a/assets/jest.config.js b/assets/jest.config.js
index 32c0a334..6251b5d2 100644
--- a/assets/jest.config.js
+++ b/assets/jest.config.js
@@ -25,7 +25,7 @@ export default {
},
preset: 'ts-jest/presets/js-with-ts-esm',
setupFilesAfterEnv: ['<rootDir>/test/jest-setup.ts'],
- testEnvironment: 'jsdom',
+ testEnvironment: './fix-jsdom.ts',
testPathIgnorePatterns: ['/node_modules/', '/dist/'],
moduleNameMapper: {
'./js/(.*)': '<rootDir>/js/$1',
diff --git a/assets/js/__tests__/ujs.spec.ts b/assets/js/__tests__/ujs.spec.ts
index 142e47c0..7f87b766 100644
--- a/assets/js/__tests__/ujs.spec.ts
+++ b/assets/js/__tests__/ujs.spec.ts
@@ -1,5 +1,5 @@
import fetchMock from 'jest-fetch-mock';
-import { fireEvent } from '@testing-library/dom';
+import { fireEvent, waitFor } from '@testing-library/dom';
import { assertType } from '../utils/assert';
import '../ujs';
@@ -199,18 +199,10 @@ describe('Remote utilities', () => {
}));
it('should reload the page on 300 multiple choices response', () => {
- const promiseLike = {
- then(cb: (r: Response) => void) {
- if (cb) {
- cb(new Response('', { status: 300 }));
- }
- }
- };
-
- jest.spyOn(global, 'fetch').mockReturnValue(promiseLike as any);
+ jest.spyOn(global, 'fetch').mockResolvedValue(new Response('', { status: 300}));
submitForm();
- expect(window.location.reload).toHaveBeenCalledTimes(1);
+ return waitFor(() => expect(window.location.reload).toHaveBeenCalledTimes(1));
});
});
});
diff --git a/assets/js/__tests__/upload-test.png b/assets/js/__tests__/upload-test.png
new file mode 100644
index 0000000000000000000000000000000000000000..770601f791c9ab903b5e3512dc49b685b638a34f
GIT binary patch
literal 527
zcmV+q0`UEbP)<h;3K|Lk000e1NJLTq00031000390ssI2kattw0004TX+uL$X=7sm
z0C?J!(K|>&Q5?tduUZsQ6jTt=a0jg=mx^dK3nVZQGtg@2K74`qD)%ZajzO!?&|}GJ
zXlbssp*0AC9uUn9O+_ssw|kIM0)zh3hyOYM!#TeL?rKiet+oK@M$wFhf>J!OB6U2|
z#vpz87?V}2FdK=4X;~k)xBzba;w=7GJzCOI!6g9!wO|$<uLIK?#e(oIFra2+&G9KP
zZaNoqd@uS+-7msvVcZCtvFM!R0YFMOS!uGGF?GjNfLGHE&2hYLO}H$q=SnK|ask~*
zV|hvRfe}J1ut)?6g%VkE6!DYAK*AzQh+vEPVZ?ld5(+u8s7TC{M@L4?BmUrbwpMC#
z0svQc5Za$?`^^U(+fb|6_UEB(*N(vR2p2|UK3|2IckomiJ?{bZZo=7Rqo?e^`4$X4
z6l7Bdzyr8bDR@1Bo&@w?L)WUv?Ps0iJBXDbd<A>^Fq($mwHEL0Y|ibkJ>U6#0m1Zg
z#*`8=)c^nh32;bRa{vG?BLDy{BLR4&KXw2B00(qQO+^Rj1Qie_2jv)t)c^nh8FWQh
zbVF}#ZDnqB07G(RVRU6=Aa`kWXdp*PO;A^X4i^9b01Qb)K~xCWWBAX&000940RR}?
Rjj#X!002ovPDHLkV1kc2)Z+jE
literal 0
HcmV?d00001
diff --git a/assets/js/__tests__/upload-test.webm b/assets/js/__tests__/upload-test.webm
new file mode 100644
index 0000000000000000000000000000000000000000..12442b6a330ba8ff13db04172b22415389c44b22
GIT binary patch
literal 555
zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7mJk;$pGkx3%BA)S!{1Q>q{`pz!d
z<-5B(cy)`Y=gPF;HH`})Jh6~<*+AY6-`zbxIiZll>A`E77&ReWnc&?($tK39Zy@F{
zM1qZ@1p#u^CavomoB5p_d>eXw63f!e4D<}m^b8FQ!W~ihE}b0?E)Yk6oPTB=)OF6+
z8ySm_c5IoqS?=_S(`)85GAM1G(_EUD($UD!)*2Qc7GT-j$f*3dxrHeyis8YO4ULSu
z8X0FbGKFsF2;JDo5W2IG2^0bj4aLO^k`FYbpP1#kxZTyy+26%A$fX_C5yi!~k`Htz
zBdkk5u@qVL44@a1fnG2+1bV?pAty7bte`@-tiUq;#6s7_9WJ3kjv=1@elG2k4GgTn
zNYB8;00J1~owt>4eBQ{gcugY%L&GA^W~T>0|FSVysK`ioF)%17FbMAd|9|<E`v3p`
iUp{3IBwsFNU;ste#NGXy8yU8D0h7hYCdTz28kqseP_%Xc
literal 0
HcmV?d00001
diff --git a/assets/js/__tests__/upload.spec.ts b/assets/js/__tests__/upload.spec.ts
new file mode 100644
index 00000000..401582af
--- /dev/null
+++ b/assets/js/__tests__/upload.spec.ts
@@ -0,0 +1,178 @@
+import { $, $$, removeEl } from '../utils/dom';
+import { assertNotNull, assertNotUndefined } from '../utils/assert';
+
+import fetchMock from 'jest-fetch-mock';
+import { fixEventListeners } from '../../test/fix-event-listeners';
+import { fireEvent, waitFor } from '@testing-library/dom';
+import { promises } from 'fs';
+import { join } from 'path';
+
+import { setupImageUpload } from '../upload';
+
+/* eslint-disable camelcase */
+const scrapeResponse = {
+ description: 'test',
+ images: [
+ {url: 'http://localhost/images/1', camo_url: 'http://localhost/images/1'},
+ {url: 'http://localhost/images/2', camo_url: 'http://localhost/images/2'},
+ ],
+ source_url: 'http://localhost/images',
+ author_name: 'test',
+};
+const nullResponse = null;
+const errorResponse = {
+ errors: ['Error 1', 'Error 2'],
+};
+/* eslint-enable camelcase */
+
+describe('Image upload form', () => {
+ let mockPng: File;
+ let mockWebm: File;
+
+ beforeAll(async() => {
+ const mockPngPath = join(__dirname, 'upload-test.png');
+ const mockWebmPath = join(__dirname, 'upload-test.webm');
+
+ mockPng = new File([(await promises.readFile(mockPngPath, { encoding: null })).buffer], 'upload-test.png', { type: 'image/png' });
+ mockWebm = new File([(await promises.readFile(mockWebmPath, { encoding: null })).buffer], 'upload-test.webm', { type: 'video/webm' });
+ });
+
+ beforeAll(() => {
+ fetchMock.enableMocks();
+ });
+
+ afterAll(() => {
+ fetchMock.disableMocks();
+ });
+
+ fixEventListeners(window);
+
+ let form: HTMLFormElement;
+ let imgPreviews: HTMLDivElement;
+ let fileField: HTMLInputElement;
+ let remoteUrl: HTMLInputElement;
+ let scraperError: HTMLDivElement;
+ let fetchButton: HTMLButtonElement;
+ let tagsEl: HTMLTextAreaElement;
+ let sourceEl: HTMLInputElement;
+ let descrEl: HTMLTextAreaElement;
+
+ beforeEach(() => {
+ document.documentElement.insertAdjacentHTML('beforeend', `
+ <form action="/images">
+ <div id="js-image-upload-previews"></div>
+ <input id="image_image" name="image[image]" type="file" class="js-scraper" />
+ <input id="image_scraper_url" name="image[scraper_url]" type="url" class="js-scraper" />
+ <button id="js-scraper-preview" type="button">Fetch</button>
+ <div class="field-error-js hidden js-scraper"></div>
+
+ <input id="image_sources_0_source" name="image[sources][0][source]" type="text" class="js-source-url" />
+ <textarea id="image_tag_input" name="image[tag_input]" class="js-image-tags-input"></textarea>
+ <textarea id="image_description" name="image[description]" class="js-image-descr-input"></textarea>
+ </form>
+ `);
+
+ form = assertNotNull($<HTMLFormElement>('form'));
+ imgPreviews = assertNotNull($<HTMLDivElement>('#js-image-upload-previews'));
+ fileField = assertNotUndefined($$<HTMLInputElement>('.js-scraper')[0]);
+ remoteUrl = assertNotUndefined($$<HTMLInputElement>('.js-scraper')[1]);
+ scraperError = assertNotUndefined($$<HTMLInputElement>('.js-scraper')[2]);
+ tagsEl = assertNotNull($<HTMLTextAreaElement>('.js-image-tags-input'));
+ sourceEl = assertNotNull($<HTMLInputElement>('.js-source-url'));
+ descrEl = assertNotNull($<HTMLTextAreaElement>('.js-image-descr-input'));
+ fetchButton = assertNotNull($<HTMLButtonElement>('#js-scraper-preview'));
+
+ setupImageUpload();
+ fetchMock.resetMocks();
+ });
+
+ afterEach(() => {
+ removeEl(form);
+ });
+
+ it('should disable fetch button on empty source', () => {
+ fireEvent.input(remoteUrl, { target: { value: '' }});
+ expect(fetchButton.disabled).toBe(true);
+ });
+
+ it('should enable fetch button on non-empty source', () => {
+ fireEvent.input(remoteUrl, { target: { value: 'http://localhost/images/1' }});
+ expect(fetchButton.disabled).toBe(false);
+ });
+
+ it('should create a preview element when an image file is uploaded', () => {
+ fireEvent.change(fileField, { target: { files: [mockPng] }});
+ return waitFor(() => expect(imgPreviews.querySelectorAll('img')).toHaveLength(1));
+ });
+
+ it('should create a preview element when a Matroska video file is uploaded', () => {
+ fireEvent.change(fileField, { target: { files: [mockWebm] }});
+ return waitFor(() => expect(imgPreviews.querySelectorAll('video')).toHaveLength(1));
+ });
+
+ it('should block navigation away after an image file is attached, but not after form submission', async() => {
+ fireEvent.change(fileField, { target: { files: [mockPng] }});
+ await waitFor(() => { expect(imgPreviews.querySelectorAll('img')).toHaveLength(1); });
+
+ const failedUnloadEvent = new Event('beforeunload', { cancelable: true });
+ expect(fireEvent(window, failedUnloadEvent)).toBe(false);
+
+ await new Promise<void>(resolve => {
+ form.addEventListener('submit', event => {
+ event.preventDefault();
+ resolve();
+ });
+ form.submit();
+ });
+
+ const succeededUnloadEvent = new Event('beforeunload', { cancelable: true });
+ expect(fireEvent(window, succeededUnloadEvent)).toBe(true);
+ });
+
+ it('should scrape images when the fetch button is clicked', async() => {
+ fetchMock.mockResolvedValue(new Response(JSON.stringify(scrapeResponse), { status: 200 }));
+ fireEvent.input(remoteUrl, { target: { value: 'http://localhost/images/1' }});
+
+ await new Promise<void>(resolve => {
+ tagsEl.addEventListener('addtag', (event: Event) => {
+ expect((event as CustomEvent).detail).toEqual({name: 'artist:test'});
+ resolve();
+ });
+
+ fireEvent.keyDown(remoteUrl, { keyCode: 13 });
+ });
+
+ await waitFor(() => expect(fetch).toHaveBeenCalledTimes(1));
+ await waitFor(() => expect(imgPreviews.querySelectorAll('img')).toHaveLength(2));
+
+ expect(scraperError.innerHTML).toEqual('');
+ expect(sourceEl.value).toEqual('http://localhost/images');
+ expect(descrEl.value).toEqual('test');
+ });
+
+ it('should show null scrape result', () => {
+ fetchMock.mockResolvedValue(new Response(JSON.stringify(nullResponse), { status: 200 }));
+
+ fireEvent.input(remoteUrl, { target: { value: 'http://localhost/images/1' }});
+ fetchButton.click();
+
+ return waitFor(() => {
+ expect(fetch).toHaveBeenCalledTimes(1);
+ expect(imgPreviews.querySelectorAll('img')).toHaveLength(0);
+ expect(scraperError.innerText).toEqual('No image found at that address.');
+ });
+ });
+
+ it('should show error scrape result', () => {
+ fetchMock.mockResolvedValue(new Response(JSON.stringify(errorResponse), { status: 200 }));
+
+ fireEvent.input(remoteUrl, { target: { value: 'http://localhost/images/1' }});
+ fetchButton.click();
+
+ return waitFor(() => {
+ expect(fetch).toHaveBeenCalledTimes(1);
+ expect(imgPreviews.querySelectorAll('img')).toHaveLength(0);
+ expect(scraperError.innerText).toEqual('Error 1 Error 2');
+ });
+ });
+});
diff --git a/assets/js/upload.js b/assets/js/upload.js
index 84482341..62f749fb 100644
--- a/assets/js/upload.js
+++ b/assets/js/upload.js
@@ -132,21 +132,17 @@ function setupImageUpload() {
});
// Enable/disable the fetch button based on content in the image scraper. Fetching with no URL makes no sense.
- remoteUrl.addEventListener('input', () => {
+ function setFetchEnabled() {
if (remoteUrl.value.length > 0) {
enableFetch();
}
else {
disableFetch();
}
- });
+ }
- if (remoteUrl.value.length > 0) {
- enableFetch();
- }
- else {
- disableFetch();
- }
+ remoteUrl.addEventListener('input', () => setFetchEnabled());
+ setFetchEnabled();
// Catch unintentional navigation away from the page
diff --git a/assets/test/fix-event-listeners.ts b/assets/test/fix-event-listeners.ts
new file mode 100644
index 00000000..d4e0a8bf
--- /dev/null
+++ b/assets/test/fix-event-listeners.ts
@@ -0,0 +1,26 @@
+// Add helper to fix event listeners on a given target
+
+export function fixEventListeners(t: EventTarget) {
+ let eventListeners: Record<string, unknown[]>;
+
+ /* eslint-disable @typescript-eslint/no-explicit-any */
+ beforeAll(() => {
+ eventListeners = {};
+ const oldAddEventListener = t.addEventListener;
+
+ t.addEventListener = function(type: string, listener: any, options: any): void {
+ eventListeners[type] = eventListeners[type] || [];
+ eventListeners[type].push(listener);
+ return oldAddEventListener(type, listener, options);
+ };
+ });
+
+ afterEach(() => {
+ for (const key in eventListeners) {
+ for (const listener of eventListeners[key]) {
+ (t.removeEventListener as any)(key, listener);
+ }
+ }
+ eventListeners = {};
+ });
+}
From 089816845e9532f562d83f89cc30ef8374b1cd93 Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Mon, 22 Apr 2024 18:45:42 -0400
Subject: [PATCH 8/8] Silence any lint on window.location mock
---
assets/js/__tests__/ujs.spec.ts | 2 ++
1 file changed, 2 insertions(+)
diff --git a/assets/js/__tests__/ujs.spec.ts b/assets/js/__tests__/ujs.spec.ts
index 7f87b766..cef79c14 100644
--- a/assets/js/__tests__/ujs.spec.ts
+++ b/assets/js/__tests__/ujs.spec.ts
@@ -117,6 +117,7 @@ describe('Remote utilities', () => {
// https://www.benmvp.com/blog/mocking-window-location-methods-jest-jsdom/
let oldWindowLocation: Location;
+ /* eslint-disable @typescript-eslint/no-explicit-any */
beforeAll(() => {
oldWindowLocation = window.location;
delete (window as any).location;
@@ -136,6 +137,7 @@ describe('Remote utilities', () => {
beforeEach(() => {
(window.location.reload as any).mockReset();
});
+ /* eslint-enable @typescript-eslint/no-explicit-any */
afterAll(() => {
// restore window.location to the jsdom Location object