From a08510f21efa7ec3a32082f70d4420b64b73c156 Mon Sep 17 00:00:00 2001 From: Yay295 Date: Mon, 28 Oct 2024 15:13:42 -0500 Subject: [PATCH 01/33] Add Bluesky scraper (#290) * add bluesky scraper * use com.atproto.sync.getBlob to get original bluesky image * ignore data after bluesky post id * fix json access and missing function * fix bluesky fullsize image regex --- lib/philomena_proxy/scrapers.ex | 1 + lib/philomena_proxy/scrapers/bluesky.ex | 48 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/philomena_proxy/scrapers/bluesky.ex diff --git a/lib/philomena_proxy/scrapers.ex b/lib/philomena_proxy/scrapers.ex index a96f0817..08674d44 100644 --- a/lib/philomena_proxy/scrapers.ex +++ b/lib/philomena_proxy/scrapers.ex @@ -21,6 +21,7 @@ defmodule PhilomenaProxy.Scrapers do } @scrapers [ + PhilomenaProxy.Scrapers.Bluesky, PhilomenaProxy.Scrapers.Deviantart, PhilomenaProxy.Scrapers.Pillowfort, PhilomenaProxy.Scrapers.Twitter, diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex new file mode 100644 index 00000000..598d1470 --- /dev/null +++ b/lib/philomena_proxy/scrapers/bluesky.ex @@ -0,0 +1,48 @@ +defmodule PhilomenaProxy.Scrapers.Bluesky do + @moduledoc false + + alias PhilomenaProxy.Scrapers.Scraper + alias PhilomenaProxy.Scrapers + + @behaviour Scraper + + @url_regex ~r|https://bsky\.app/profile/([^/]+)/post/([^/?#]+)| + @fullsize_image_regex ~r|.*/img/feed_fullsize/plain/([^/]+)/([^@]+).*| + @blob_image_url_pattern "https://bsky.social/xrpc/com.atproto.sync.getBlob/?did=\\1&cid=\\2" + + @spec can_handle?(URI.t(), String.t()) :: boolean() + def can_handle?(_uri, url) do + String.match?(url, @url_regex) + end + + @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result() + def scrape(_uri, url) do + [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) + + api_url_resolve_handle = + "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" + + did = PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did) + + api_url_get_posts = + "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" + + post_json = PhilomenaProxy.Http.get(api_url_get_posts) |> json!() |> Map.fetch!(:posts) |> hd + + %{ + source_url: url, + author_name: post_json["author"]["handle"], + description: post_json["record"]["text"], + images: + post_json["embed"]["images"] + |> Enum.map( + &%{ + url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern), + camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"]) + } + ) + } + end + + defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body) +end From c2cb6a1649a22ab4657f35f4117b0df0efb14f8e Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 7 Nov 2024 13:46:16 -0500 Subject: [PATCH 02/33] Change PhilomenaQuery.Batch to Stream-based instead of callback-based --- lib/mix/tasks/upload_to_s3.ex | 8 +- lib/philomena/user_downvote_wipe.ex | 9 +- .../workers/tag_change_revert_worker.ex | 4 +- lib/philomena_query/batch.ex | 93 ++++++++++--------- lib/philomena_query/search.ex | 6 +- 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex index 87ab48d3..61c61f87 100644 --- a/lib/mix/tasks/upload_to_s3.ex +++ b/lib/mix/tasks/upload_to_s3.ex @@ -121,7 +121,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name), timeout: :infinity @@ -142,7 +144,9 @@ defmodule Mix.Tasks.UploadToS3 do end defp upload_images(queryable, batch_size, file_root, new_file_root) do - Batch.record_batches(queryable, [batch_size: batch_size], fn models -> + queryable + |> Batch.record_batches(batch_size: batch_size) + |> Enum.each(fn models -> models |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity) |> Stream.run() diff --git a/lib/philomena/user_downvote_wipe.ex b/lib/philomena/user_downvote_wipe.ex index 6fb217fe..b958ab9b 100644 --- a/lib/philomena/user_downvote_wipe.ex +++ b/lib/philomena/user_downvote_wipe.ex @@ -15,7 +15,8 @@ defmodule Philomena.UserDownvoteWipe do ImageVote |> where(user_id: ^user.id, up: false) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -31,7 +32,8 @@ defmodule Philomena.UserDownvoteWipe do if upvotes_and_faves_too do ImageVote |> where(user_id: ^user.id, up: true) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_v], i_v.image_id)) {count, nil} = @@ -46,7 +48,8 @@ defmodule Philomena.UserDownvoteWipe do ImageFave |> where(user_id: ^user.id) - |> Batch.query_batches([id_field: :image_id], fn queryable -> + |> Batch.query_batches(id_field: :image_id) + |> Enum.each(fn queryable -> {_, image_ids} = Repo.delete_all(select(queryable, [i_f], i_f.image_id)) {count, nil} = diff --git a/lib/philomena/workers/tag_change_revert_worker.ex b/lib/philomena/workers/tag_change_revert_worker.ex index 519b8404..80058a31 100644 --- a/lib/philomena/workers/tag_change_revert_worker.ex +++ b/lib/philomena/workers/tag_change_revert_worker.ex @@ -27,7 +27,9 @@ defmodule Philomena.TagChangeRevertWorker do batch_size = attributes["batch_size"] || 100 attributes = Map.delete(attributes, "batch_size") - Batch.query_batches(queryable, [batch_size: batch_size], fn queryable -> + queryable + |> Batch.query_batches(batch_size: batch_size) + |> Enum.each(fn queryable -> ids = Repo.all(select(queryable, [tc], tc.id)) TagChanges.mass_revert(ids, cast_ip(atomify_keys(attributes))) end) diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex index 918a3b5e..20bdc364 100644 --- a/lib/philomena_query/batch.ex +++ b/lib/philomena_query/batch.ex @@ -25,24 +25,32 @@ defmodule PhilomenaQuery.Batch do @type id_field :: {:id_field, atom()} @type batch_options :: [batch_size() | id_field()] - @typedoc """ - The callback for `record_batches/3`. + @doc """ + Stream schema structures on a queryable, using batches to avoid locking. + + Valid options: + * `batch_size` (integer) - the number of records to load per batch + * `id_field` (atom) - the name of the field containing the ID + + ## Example + + queryable = from i in Image, where: i.image_width >= 1920 + + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(fn image -> IO.inspect(image.id) end) - Takes a list of schema structs which were returned in the batch. Return value is ignored. """ - @type record_batch_callback :: ([struct()] -> any()) - - @typedoc """ - The callback for `query_batches/3`. - - Takes an `m:Ecto.Query` that can be processed with `m:Philomena.Repo` query commands, such - as `Philomena.Repo.update_all/3` or `Philomena.Repo.delete_all/2`. Return value is ignored. - """ - @type query_batch_callback :: ([Ecto.Query.t()] -> any()) + @spec records(queryable(), batch_options()) :: Enumerable.t() + def records(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.flat_map(&Repo.all/1) + end @doc """ - Execute a callback with lists of schema structures on a queryable, - using batches to avoid locking. + Stream lists of schema structures on a queryable, using batches to avoid + locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -56,16 +64,20 @@ defmodule PhilomenaQuery.Batch do Enum.each(images, &IO.inspect(&1.id)) end - PhilomenaQuery.Batch.record_batches(queryable, cb) + queryable + |> PhilomenaQuery.Batch.record_batches() + |> Enum.each(cb) """ - @spec record_batches(queryable(), batch_options(), record_batch_callback()) :: [] - def record_batches(queryable, opts \\ [], callback) do - query_batches(queryable, opts, &callback.(Repo.all(&1))) + @spec record_batches(queryable(), batch_options()) :: Enumerable.t() + def record_batches(queryable, opts \\ []) do + queryable + |> query_batches(opts) + |> Stream.map(&Repo.all/1) end @doc """ - Execute a callback with bulk queries on a queryable, using batches to avoid locking. + Stream bulk queries on a queryable, using batches to avoid locking. Valid options: * `batch_size` (integer) - the number of records to load per batch @@ -76,41 +88,36 @@ defmodule PhilomenaQuery.Batch do > If you are looking to receive schema structures (e.g., you are querying for `Image`s, > and you want to receive `Image` objects, then use `record_batches/3` instead. - An `m:Ecto.Query` which selects all IDs in the current batch is passed into the callback - during each invocation. + `m:Ecto.Query` structs which select the IDs in each batch are streamed out. ## Example queryable = from ui in ImageVote, where: ui.user_id == 1234 - opts = [id_field: :image_id] - - cb = fn bulk_query -> - Repo.delete_all(bulk_query) - end - - PhilomenaQuery.Batch.query_batches(queryable, opts, cb) + queryable + |> PhilomenaQuery.Batch.query_batches(id_field: :image_id) + |> Enum.each(fn batch_query -> Repo.delete_all(batch_query) end) """ - @spec query_batches(queryable(), batch_options(), query_batch_callback()) :: [] - def query_batches(queryable, opts \\ [], callback) do - ids = load_ids(queryable, -1, opts) - - query_batches(queryable, opts, callback, ids) - end - - defp query_batches(_queryable, _opts, _callback, []), do: [] - - defp query_batches(queryable, opts, callback, ids) do + @spec query_batches(queryable(), batch_options()) :: Enumerable.t(Ecto.Query.t()) + def query_batches(queryable, opts \\ []) do id_field = Keyword.get(opts, :id_field, :id) - queryable - |> where([m], field(m, ^id_field) in ^ids) - |> callback.() + Stream.unfold( + load_ids(queryable, -1, opts), + fn + [] -> + # Stop when no more results are produced + nil - ids = load_ids(queryable, Enum.max(ids), opts) + ids -> + # Process results and output next query + output = where(queryable, [m], field(m, ^id_field) in ^ids) + next_ids = load_ids(queryable, Enum.max(ids), opts) - query_batches(queryable, opts, callback, ids) + {output, next_ids} + end + ) end defp load_ids(queryable, max_id, opts) do diff --git a/lib/philomena_query/search.ex b/lib/philomena_query/search.ex index 2519e580..cd02137c 100644 --- a/lib/philomena_query/search.ex +++ b/lib/philomena_query/search.ex @@ -199,11 +199,13 @@ defmodule PhilomenaQuery.Search do Search.reindex(query, Image, batch_size: 5000) """ - @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: [] + @spec reindex(queryable(), schema_module(), Batch.batch_options()) :: :ok def reindex(queryable, module, opts \\ []) do index = @policy.index_for(module) - Batch.record_batches(queryable, opts, fn records -> + queryable + |> Batch.record_batches(opts) + |> Enum.each(fn records -> lines = Enum.flat_map(records, fn record -> doc = index.as_json(record) From 7312c647a810b84d1359622b2d8666f159b280f5 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:26:51 -0500 Subject: [PATCH 03/33] Add additional linting workflows --- .github/workflows/elixir.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index a92aa72f..734c0b68 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -31,6 +31,9 @@ jobs: - run: docker compose pull - run: docker compose build + - name: mix format + run: docker compose run app mix format --check-formatted + - name: Build and test run: docker compose run app run-test @@ -50,6 +53,21 @@ jobs: - uses: actions/checkout@v4 - uses: crate-ci/typos@master + cargo: + name: Rust Linting and Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: cargo fmt + run: (cd native/philomena && cargo fmt --check) + + - name: cargo clippy + run: (cd native/philomena && cargo clippy -- -D warnings) + lint-and-test: name: 'JavaScript Linting and Unit Tests' runs-on: ubuntu-latest @@ -80,4 +98,4 @@ jobs: working-directory: ./assets - run: npm run build - working-directory: ./assets \ No newline at end of file + working-directory: ./assets From 5fab189f82379df3d92c49660ef53c42a2e21c59 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:27:46 -0500 Subject: [PATCH 04/33] Run cargo fmt --- native/philomena/src/camo.rs | 4 ++-- native/philomena/src/markdown.rs | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index 5a72b3f6..df47a2e8 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -1,8 +1,8 @@ +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; use ring::hmac; use std::env; use url::Url; -use base64::Engine; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; fn trusted_host(mut url: Url) -> Option { url.set_port(Some(443)).ok()?; diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index af976233..c92e6a0f 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,5 +1,5 @@ -use comrak::ComrakOptions; use crate::camo; +use comrak::ComrakOptions; use rustler::{MapIterator, Term}; use std::collections::HashMap; use std::env; @@ -19,19 +19,28 @@ fn common_options() -> ComrakOptions { options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); if let Ok(domains) = env::var("SITE_DOMAINS") { - options.extension.philomena_domains = Some(domains.split(',').map(|s| s.to_string()).collect::>()); + options.extension.philomena_domains = Some( + domains + .split(',') + .map(|s| s.to_string()) + .collect::>(), + ); } options } fn map_to_hashmap(map: Term) -> Option> { - Some(MapIterator::new(map)?.map(|(key, value)| { - let key: String = key.decode().unwrap_or_else(|_| String::from("")); - let value: String = value.decode().unwrap_or_else(|_| String::from("")); + Some( + MapIterator::new(map)? + .map(|(key, value)| { + let key: String = key.decode().unwrap_or_else(|_| String::from("")); + let value: String = value.decode().unwrap_or_else(|_| String::from("")); - (key, value) - }).collect()) + (key, value) + }) + .collect(), + ) } pub fn to_html(input: String, reps: Term) -> String { From 9c797e8d643c645b975ccd76a1bb0c7b96faf4fe Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 8 Nov 2024 23:55:50 -0500 Subject: [PATCH 05/33] Rust code refactor --- lib/philomena_web/markdown_renderer.ex | 6 ++-- native/philomena/Cargo.lock | 2 ++ native/philomena/Cargo.toml | 10 +++--- native/philomena/src/camo.rs | 20 ++++++----- native/philomena/src/domains.rs | 14 ++++++++ native/philomena/src/lib.rs | 11 +++--- native/philomena/src/markdown.rs | 49 +++++++------------------- 7 files changed, 54 insertions(+), 58 deletions(-) create mode 100644 native/philomena/src/domains.rs diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex index 7caff5c9..0b108008 100644 --- a/lib/philomena_web/markdown_renderer.ex +++ b/lib/philomena_web/markdown_renderer.ex @@ -75,8 +75,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do defp render_representations(images, conn) do loaded_images = load_images(images) - images - |> Enum.map(fn group -> + Map.new(images, fn group -> img = loaded_images[Enum.at(group, 0)] text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}" @@ -131,8 +130,7 @@ defmodule PhilomenaWeb.MarkdownRenderer do |> Phoenix.HTML.Safe.to_iodata() |> IO.iodata_to_binary() - [text, string_contents] + {text, string_contents} end) - |> Map.new(fn [id, html] -> {id, html} end) end end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 0ca74719..495a6815 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -260,7 +260,9 @@ version = "0.3.0" dependencies = [ "base64", "comrak", + "http", "jemallocator", + "regex", "ring", "rustler", "url", diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 942d39bc..20cb3b9b 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -10,11 +10,13 @@ path = "src/lib.rs" crate-type = ["dylib"] [dependencies] -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } -jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } -rustler = "0.28" -ring = "0.16" base64 = "0.21" +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } +http = "0.2" +jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } +regex = "1" +ring = "0.16" +rustler = "0.28" url = "2.3" [profile.release] diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index df47a2e8..c79f12e1 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -11,7 +11,7 @@ fn trusted_host(mut url: Url) -> Option { Some(url.to_string()) } -fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option { +fn untrusted_host(url: Url, camo_host: &str, camo_key: &str) -> Option { let camo_url = format!("https://{}", camo_host); let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref()); let tag = hmac::sign(&key, url.to_string().as_bytes()); @@ -27,20 +27,24 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option Option { +pub fn image_url(uri: &str) -> Option { let cdn_host = env::var("CDN_HOST").ok()?; - let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| String::from("")); - let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| String::from("")); + let camo_host = env::var("CAMO_HOST").unwrap_or_else(|_| "".into()); + let camo_key = env::var("CAMO_KEY").unwrap_or_else(|_| "".into()); if camo_key.is_empty() { - return Some(uri); + return Some(uri.into()); } - let url = Url::parse(&uri).ok()?; + let url = Url::parse(uri).ok()?; match url.host_str() { Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url), - Some(_) => untrusted_host(url, camo_host, camo_key), - None => Some(String::from("")), + Some(_) => untrusted_host(url, &camo_host, &camo_key), + None => Some("".into()), } } + +pub fn image_url_careful(uri: &str) -> String { + image_url(uri).unwrap_or_else(|| "".into()) +} diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs new file mode 100644 index 00000000..c90ddb6f --- /dev/null +++ b/native/philomena/src/domains.rs @@ -0,0 +1,14 @@ +use std::env; + +pub fn get() -> Option> { + if let Ok(domains) = env::var("SITE_DOMAINS") { + return Some( + domains + .split(',') + .map(|s| s.to_string()) + .collect::>(), + ); + } + + None +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index f5317aa3..bcbadcb6 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,7 +1,8 @@ use jemallocator::Jemalloc; -use rustler::Term; +use std::collections::HashMap; mod camo; +mod domains; mod markdown; #[global_allocator] @@ -15,18 +16,18 @@ rustler::init! { // Markdown NIF wrappers. #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html(input: String, reps: Term) -> String { +fn markdown_to_html(input: &str, reps: HashMap) -> String { markdown::to_html(input, reps) } #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html_unsafe(input: String, reps: Term) -> String { +fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String { markdown::to_html_unsafe(input, reps) } // Camo NIF wrappers. #[rustler::nif] -fn camo_image_url(input: String) -> String { - camo::image_url(input).unwrap_or_else(|| String::from("")) +fn camo_image_url(input: &str) -> String { + camo::image_url_careful(input) } diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index c92e6a0f..a927a6e5 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,11 +1,9 @@ -use crate::camo; -use comrak::ComrakOptions; -use rustler::{MapIterator, Term}; +use crate::{camo, domains}; +use comrak::Options; use std::collections::HashMap; -use std::env; -fn common_options() -> ComrakOptions { - let mut options = ComrakOptions::default(); +fn common_options() -> Options { + let mut options = Options::default(); options.extension.autolink = true; options.extension.table = true; options.extension.description_lists = true; @@ -16,47 +14,24 @@ fn common_options() -> ComrakOptions { options.render.hardbreaks = true; options.render.github_pre_lang = true; - options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); - - if let Ok(domains) = env::var("SITE_DOMAINS") { - options.extension.philomena_domains = Some( - domains - .split(',') - .map(|s| s.to_string()) - .collect::>(), - ); - } + options.extension.camoifier = Some(|s| camo::image_url_careful(&s)); + options.extension.philomena_domains = domains::get(); options } -fn map_to_hashmap(map: Term) -> Option> { - Some( - MapIterator::new(map)? - .map(|(key, value)| { - let key: String = key.decode().unwrap_or_else(|_| String::from("")); - let value: String = value.decode().unwrap_or_else(|_| String::from("")); - - (key, value) - }) - .collect(), - ) -} - -pub fn to_html(input: String, reps: Term) -> String { +pub fn to_html(input: &str, reps: HashMap) -> String { let mut options = common_options(); options.render.escape = true; + options.extension.philomena_replacements = Some(reps); - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) + comrak::markdown_to_html(input, &options) } -pub fn to_html_unsafe(input: String, reps: Term) -> String { +pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { let mut options = common_options(); options.render.unsafe_ = true; + options.extension.philomena_replacements = Some(reps); - options.extension.philomena_replacements = map_to_hashmap(reps); - - comrak::markdown_to_html(&input, &options) + comrak::markdown_to_html(input, &options) } From 1de4a7b82011ec73a38d939710e0e1750a424000 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 3 Nov 2024 19:17:01 -0500 Subject: [PATCH 06/33] Add framework for zip data export --- lib/philomena/data_exports/aggregator.ex | 171 ++++++++++++++++++++ lib/philomena/data_exports/zip_generator.ex | 56 +++++++ lib/philomena/native.ex | 12 ++ lib/philomena_query/batch.ex | 3 +- native/philomena/Cargo.lock | 169 ++++++++++++++++++- native/philomena/Cargo.toml | 1 + native/philomena/src/lib.rs | 36 ++++- native/philomena/src/zip.rs | 69 ++++++++ 8 files changed, 509 insertions(+), 8 deletions(-) create mode 100644 lib/philomena/data_exports/aggregator.ex create mode 100644 lib/philomena/data_exports/zip_generator.ex create mode 100644 native/philomena/src/zip.rs diff --git a/lib/philomena/data_exports/aggregator.ex b/lib/philomena/data_exports/aggregator.ex new file mode 100644 index 00000000..5dcb82b3 --- /dev/null +++ b/lib/philomena/data_exports/aggregator.ex @@ -0,0 +1,171 @@ +defmodule Philomena.DataExports.Aggregator do + @moduledoc """ + Data generation module for data export logic. + """ + + import Ecto.Query + alias PhilomenaQuery.Batch + + # Direct PII + alias Philomena.Donations.Donation + alias Philomena.UserFingerprints.UserFingerprint + alias Philomena.UserIps.UserIp + alias Philomena.UserNameChanges.UserNameChange + alias Philomena.Users.User + + # UGC for export + alias Philomena.ArtistLinks.ArtistLink + alias Philomena.Badges.Award + alias Philomena.Comments.Comment + alias Philomena.Commissions.Commission + alias Philomena.DnpEntries.DnpEntry + alias Philomena.DuplicateReports.DuplicateReport + alias Philomena.Filters.Filter + alias Philomena.ImageFaves.ImageFave + alias Philomena.ImageHides.ImageHide + alias Philomena.ImageVotes.ImageVote + alias Philomena.Images.Image + alias Philomena.PollVotes.PollVote + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.SourceChanges.SourceChange + alias Philomena.TagChanges.TagChange + alias Philomena.Topics.Topic + alias Philomena.Bans.User, as: UserBan + + # Direct UGC from form submission + @user_columns [ + :created_at, + :name, + :email, + :description, + :current_filter_id, + :spoiler_type, + :theme, + :images_per_page, + :show_large_thumbnails, + :show_sidebar_and_watched_images, + :fancy_tag_field_on_upload, + :fancy_tag_field_on_edit, + :fancy_tag_field_in_settings, + :autorefresh_by_default, + :anonymous_by_default, + :comments_newest_first, + :comments_always_jump_to_last, + :comments_per_page, + :watch_on_reply, + :watch_on_new_topic, + :watch_on_upload, + :messages_newest_first, + :serve_webm, + :no_spoilered_in_watched, + :watched_images_query_str, + :watched_images_exclude_str, + :use_centered_layout, + :personal_title, + :hide_vote_counts, + :scale_large_images + ] + + # All these also have created_at and are selected by user_id + @indirect_columns [ + {Donation, [:email, :amount, :fee, :note]}, + {UserFingerprint, [:fingerprint, :uses, :updated_at]}, + {UserIp, [:ip, :uses, :updated_at]}, + {UserNameChange, [:name]}, + {ArtistLink, [:aasm_state, :uri, :public, :tag_id]}, + {Award, [:label, :badge_name, :badge_id]}, + {Comment, + [ + :ip, + :fingerprint, + :user_agent, + :referrer, + :anonymous, + :image_id, + :edited_at, + :edit_reason, + :body + ]}, + {Commission, + [:open, :sheet_image_id, :categories, :information, :contact, :will_create, :will_not_create]}, + {DnpEntry, [:tag_id, :aasm_state, :dnp_type, :hide_reason, :feedback, :reason, :instructions], + :requesting_user_id}, + {DuplicateReport, [:reason, :image_id, :duplicate_of_image_id]}, + {Filter, + [ + :name, + :description, + :public, + :hidden_complex_str, + :spoilered_complex_str, + :hidden_tag_ids, + :spoilered_tag_ids + ]}, + {ImageFave, [:image_id], :user_id, :image_id}, + {ImageHide, [:image_id], :user_id, :image_id}, + {ImageVote, [:image_id, :up], :user_id, :image_id}, + {Image, [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :description]}, + {PollVote, [:rank, :poll_option_id]}, + {Post, + [:ip, :fingerprint, :user_agent, :referrer, :anonymous, :edited_at, :edit_reason, :body]}, + {Report, + [:ip, :fingerprint, :user_agent, :referrer, :reason, :reportable_id, :reportable_type]}, + {SourceChange, [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :value]}, + {TagChange, + [:ip, :fingerprint, :user_agent, :referrer, :image_id, :added, :tag_id, :tag_name_cache]}, + {Topic, [:title, :anonymous, :forum_id]}, + {UserBan, [:reason, :generated_ban_id]} + ] + + @doc """ + Get all of the export data for the given user. + """ + def get_for_user(user_id) do + [select_user(user_id)] ++ select_indirect(user_id) + end + + defp select_user(user_id) do + select_schema_by_key(user_id, User, @user_columns, :id) + end + + defp select_indirect(user_id) do + Enum.map(@indirect_columns, fn + {schema_name, columns} -> + select_schema_by_key(user_id, schema_name, columns) + + {schema_name, columns, key_column} -> + select_schema_by_key(user_id, schema_name, columns, key_column) + + {schema_name, columns, key_column, id_field} -> + select_schema_by_key(user_id, schema_name, columns, key_column, id_field) + end) + end + + defp select_schema_by_key( + user_id, + schema_name, + columns, + key_column \\ :user_id, + id_field \\ :id + ) do + table_name = schema_name.__schema__(:source) + columns = [:created_at] ++ columns + + {"#{table_name}.jsonl", + schema_name + |> where([s], field(s, ^key_column) == ^user_id) + |> select([s], map(s, ^columns)) + |> Batch.records(id_field: id_field) + |> results_as_json_lines()} + end + + defp results_as_json_lines(list_of_maps) do + Stream.map(list_of_maps, fn map -> + map + |> Map.new(fn {k, v} -> {k, to_string(v)} end) + |> Jason.encode!() + |> Kernel.<>("\n") + end) + end +end diff --git a/lib/philomena/data_exports/zip_generator.ex b/lib/philomena/data_exports/zip_generator.ex new file mode 100644 index 00000000..8c5aaf7e --- /dev/null +++ b/lib/philomena/data_exports/zip_generator.ex @@ -0,0 +1,56 @@ +defmodule Philomena.DataExports.ZipGenerator do + @moduledoc """ + ZIP file generator for an export. + """ + + alias Philomena.Native + + @doc """ + Write the ZIP file for the given aggregate data. + + Expects a list of 2-tuples, with the first element being the name of the + file to generate, and the second element being a stream which generates the + binary contents of the file. + """ + @spec generate(Path.t(), Enumerable.t()) :: :ok | atom() + def generate(filename, aggregate) do + case Native.zip_open_writer(filename) do + {:ok, zip} -> + stream_aggregate(zip, aggregate) + + error -> + error + end + end + + @spec stream_aggregate(reference(), Enumerable.t()) :: {:ok, reference()} | :error + defp stream_aggregate(zip, aggregate) do + aggregate + |> Enum.reduce_while(:ok, fn {name, content_stream}, _ -> + with :ok <- Native.zip_start_file(zip, name), + :ok <- stream_file_data(zip, content_stream) do + {:cont, :ok} + else + error -> + {:halt, error} + end + end) + |> case do + :ok -> + Native.zip_finish(zip) + + error -> + error + end + end + + @spec stream_file_data(reference(), Enumerable.t(iodata())) :: :ok | :error + defp stream_file_data(zip, content_stream) do + Enum.reduce_while(content_stream, :ok, fn iodata, _ -> + case Native.zip_write(zip, IO.iodata_to_binary(iodata)) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end +end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index a67dc33f..14eeaa17 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -11,4 +11,16 @@ defmodule Philomena.Native do @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()} + def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_start_file(reference(), String.t()) :: :ok | :error + def zip_start_file(_zip, _name), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_write(reference(), binary()) :: :ok | :error + def zip_write(_zip, _data), do: :erlang.nif_error(:nif_not_loaded) + + @spec zip_finish(reference()) :: :ok | :error + def zip_finish(_zip), do: :erlang.nif_error(:nif_not_loaded) end diff --git a/lib/philomena_query/batch.ex b/lib/philomena_query/batch.ex index 20bdc364..bb9c2c74 100644 --- a/lib/philomena_query/batch.ex +++ b/lib/philomena_query/batch.ex @@ -125,8 +125,9 @@ defmodule PhilomenaQuery.Batch do batch_size = Keyword.get(opts, :batch_size, 1000) queryable - |> exclude(:preload) |> exclude(:order_by) + |> exclude(:preload) + |> exclude(:select) |> order_by(asc: ^id_field) |> where([m], field(m, ^id_field) > ^max_id) |> select([m], field(m, ^id_field)) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 495a6815..623227a1 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -11,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "base64" version = "0.21.7" @@ -57,6 +72,21 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "darling" version = "0.20.9" @@ -92,6 +122,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_builder" version = "0.20.0" @@ -129,12 +170,39 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322ef0094744e63628e6f0eb2295517f79276a5b342a4c2ff3042566ca181d4e" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "entities" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -150,6 +218,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + [[package]] name = "heck" version = "0.4.1" @@ -183,6 +257,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.11" @@ -230,6 +314,12 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -238,9 +328,18 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] [[package]] name = "once_cell" @@ -266,13 +365,14 @@ dependencies = [ "ring", "rustler", "url", + "zip", ] [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -363,6 +463,12 @@ dependencies = [ "unreachable", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slug" version = "0.1.5" @@ -387,15 +493,35 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.59" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -561,3 +687,34 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 20cb3b9b..9ef7caac 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -18,6 +18,7 @@ regex = "1" ring = "0.16" rustler = "0.28" url = "2.3" +zip = { version = "2.2.0", features = ["deflate"], default-features = false } [profile.release] opt-level = 3 diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index bcbadcb6..6c8f96f7 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,16 +1,28 @@ use jemallocator::Jemalloc; +use rustler::{Atom, Binary, Env, Term}; use std::collections::HashMap; mod camo; mod domains; mod markdown; +mod zip; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", - [markdown_to_html, markdown_to_html_unsafe, camo_image_url] + [ + markdown_to_html, markdown_to_html_unsafe, camo_image_url, + zip_open_writer, zip_start_file, zip_write, zip_finish + ], + load = load +} + +// Setup. + +fn load(env: Env, arg: Term) -> bool { + zip::load(env, arg) } // Markdown NIF wrappers. @@ -31,3 +43,25 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String fn camo_image_url(input: &str) -> String { camo::image_url_careful(input) } + +// Zip NIF wrappers. + +#[rustler::nif] +fn zip_open_writer(path: &str) -> Result { + zip::open_writer(path) +} + +#[rustler::nif] +fn zip_start_file(writer: zip::WriterResourceArc, name: &str) -> Atom { + zip::start_file(writer, name) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_write(writer: zip::WriterResourceArc, data: Binary) -> Atom { + zip::write(writer, data.as_slice()) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn zip_finish(writer: zip::WriterResourceArc) -> Atom { + zip::finish(writer) +} diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs new file mode 100644 index 00000000..60ef15be --- /dev/null +++ b/native/philomena/src/zip.rs @@ -0,0 +1,69 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::sync::Mutex; + +use rustler::{Atom, Env, ResourceArc, Term}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + +mod atoms { + rustler::atoms! { + ok, + error, + } +} + +pub struct WriterResource { + inner: Mutex>>, +} + +pub type WriterResourceArc = ResourceArc; + +pub fn load(env: Env, _: Term) -> bool { + rustler::resource!(WriterResource, env); + true +} + +fn with_writer(writer: WriterResourceArc, f: F) -> Atom +where + F: FnOnce(&mut Option>) -> Option, +{ + let mut guard = match (*writer).inner.lock() { + Ok(g) => g, + Err(_) => return atoms::error(), + }; + + match f(&mut guard) { + Some(_) => atoms::ok(), + None => atoms::error(), + } +} + +pub fn open_writer(path: &str) -> Result { + match OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(path) + { + Ok(file) => Ok(ResourceArc::new(WriterResource { + inner: Mutex::new(Some(ZipWriter::new(file))), + })), + Err(_) => Err(atoms::error()), + } +} + +pub fn start_file(writer: WriterResourceArc, name: &str) -> Atom { + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + + with_writer(writer, move |writer| { + writer.as_mut()?.start_file(name, options).ok() + }) +} + +pub fn write(writer: WriterResourceArc, data: &[u8]) -> Atom { + with_writer(writer, move |writer| writer.as_mut()?.write(data).ok()) +} + +pub fn finish(writer: WriterResourceArc) -> Atom { + with_writer(writer, move |writer| writer.take().map(|w| w.finish().ok())) +} From a21189ae10634823d3ed91f3044bfacde14d3b32 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 00:11:08 -0500 Subject: [PATCH 07/33] Update to Philomena comrak v0.29.0 --- .github/workflows/elixir.yml | 3 + native/philomena/Cargo.lock | 90 ++++++----- native/philomena/Cargo.toml | 2 +- native/philomena/src/domains.rs | 20 +++ native/philomena/src/lib.rs | 2 + native/philomena/src/markdown.rs | 31 +++- native/philomena/src/tests.rs | 257 +++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 49 deletions(-) create mode 100644 native/philomena/src/tests.rs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 734c0b68..38b201d4 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -68,6 +68,9 @@ jobs: - name: cargo clippy run: (cd native/philomena && cargo clippy -- -D warnings) + - name: cargo test + run: (cd native/philomena && cargo test) + lint-and-test: name: 'JavaScript Linting and Unit Tests' runs-on: ubuntu-latest diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 623227a1..bc4af0b4 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -32,6 +32,29 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "bon" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663" +dependencies = [ + "darling", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -44,6 +67,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "caseless" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" +dependencies = [ + "regex", + "unicode-normalization", +] + [[package]] name = "cc" version = "1.0.94" @@ -58,12 +91,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" -version = "0.24.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a" +version = "0.29.0" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e" dependencies = [ - "derive_builder", + "bon", + "caseless", "entities", - "http", "memchr", "once_cell", "regex", @@ -89,9 +122,9 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -99,9 +132,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", @@ -113,9 +146,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", @@ -133,37 +166,6 @@ dependencies = [ "syn", ] -[[package]] -name = "derive_builder" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" -dependencies = [ - "derive_builder_core", - "syn", -] - [[package]] name = "deunicode" version = "1.4.4" @@ -463,6 +465,12 @@ dependencies = [ "unreachable", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "simd-adler32" version = "0.3.7" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 9ef7caac..19d68342 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["dylib"] [dependencies] base64 = "0.21" -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs index c90ddb6f..c5626c12 100644 --- a/native/philomena/src/domains.rs +++ b/native/philomena/src/domains.rs @@ -1,3 +1,5 @@ +use http::Uri; +use regex::Regex; use std::env; pub fn get() -> Option> { @@ -12,3 +14,21 @@ pub fn get() -> Option> { None } + +pub fn relativize(domains: &[String], url: &str) -> Option { + let uri = url.parse::().ok()?; + + if let Some(a) = uri.authority() { + if domains.contains(&a.host().to_string()) { + if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) { + return Some(re.replace(url, "").into()); + } + } + } + + Some(url.into()) +} + +pub fn relativize_careful(domains: &[String], url: &str) -> String { + relativize(domains, url).unwrap_or_else(|| url.into()) +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 6c8f96f7..ccca12a0 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -5,6 +5,8 @@ use std::collections::HashMap; mod camo; mod domains; mod markdown; +#[cfg(test)] +mod tests; mod zip; #[global_allocator] diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index a927a6e5..778deb95 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,37 +1,54 @@ use crate::{camo, domains}; use comrak::Options; use std::collections::HashMap; +use std::sync::Arc; -fn common_options() -> Options { +pub fn common_options() -> Options { let mut options = Options::default(); + + // Upstream options options.extension.autolink = true; options.extension.table = true; options.extension.description_lists = true; options.extension.superscript = true; options.extension.strikethrough = true; - options.extension.philomena = true; options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; + options.render.escape = true; - options.extension.camoifier = Some(|s| camo::image_url_careful(&s)); - options.extension.philomena_domains = domains::get(); + // Philomena options + options.extension.underline = true; + options.extension.spoiler = true; + options.extension.greentext = true; + options.extension.subscript = true; + options.extension.philomena = true; + options.render.ignore_empty_links = true; + options.render.ignore_setext = true; + + options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url))); + + if let Some(domains) = domains::get() { + options.extension.link_url_rewriter = Some(Arc::new(move |url: &str| { + domains::relativize_careful(&domains, url) + })); + } options } pub fn to_html(input: &str, reps: HashMap) -> String { let mut options = common_options(); - options.render.escape = true; - options.extension.philomena_replacements = Some(reps); + options.extension.replacements = Some(reps); comrak::markdown_to_html(input, &options) } pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { let mut options = common_options(); + options.render.escape = false; options.render.unsafe_ = true; - options.extension.philomena_replacements = Some(reps); + options.extension.replacements = Some(reps); comrak::markdown_to_html(input, &options) } diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs new file mode 100644 index 00000000..9f1f963f --- /dev/null +++ b/native/philomena/src/tests.rs @@ -0,0 +1,257 @@ +use std::{collections::HashMap, sync::Arc}; + +use crate::{domains, markdown::*}; + +fn test_options() -> comrak::Options { + let mut options = common_options(); + options.extension.image_url_rewriter = None; + options.extension.link_url_rewriter = None; + options +} + +fn html(input: &str, expected: &str) { + html_opts_w(input, expected, &test_options()); +} + +fn html_opts_i(input: &str, expected: &str, opts: F) +where + F: Fn(&mut comrak::Options), +{ + let mut options = test_options(); + opts(&mut options); + + html_opts_w(input, expected, &options); +} + +fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) { + let output = comrak::markdown_to_html(input, options); + + if output != expected { + println!("Input:"); + println!("========================"); + println!("{}", input); + println!("========================"); + println!("Expected:"); + println!("========================"); + println!("{}", expected); + println!("========================"); + println!("Output:"); + println!("========================"); + println!("{}", output); + println!("========================"); + } + assert_eq!(output, expected); +} + +#[test] +fn subscript() { + html("H%2%O\n", "
H2O
\n"); +} + +#[test] +fn spoiler() { + html( + "The ||dog dies at the end of Marley and Me||.\n", + "
The dog dies at the end of Marley and Me.
\n", + ); +} + +#[test] +fn spoiler_in_table() { + html( + "Text | Result\n--- | ---\n`||some clever text||` | ||some clever text||\n", + concat!( + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
TextResult
||some clever text||some clever text
\n" + ), + ); +} + +#[test] +fn spoiler_regressions() { + html( + "|should not be spoiler|\n||should be spoiler||\n|||should be spoiler surrounded by pipes|||", + concat!( + "
|should not be spoiler|
\n", + "should be spoiler
\n", + "|should be spoiler surrounded by pipes|
\n" + ), + ); +} + +#[test] +fn mismatched_spoilers() { + html( + "|||this is a spoiler with pipe in front||\n||this is not a spoiler|\n||this is a spoiler with pipe after|||", + concat!( + "
|this is a spoiler with pipe in front
\n", + "||this is not a spoiler|
\n", + "this is a spoiler with pipe after|
\n" + ), + ); +} + +#[test] +fn underline() { + html( + "__underlined__\n", + "
underlined
\n", + ); +} + +#[test] +fn no_setext_headings_in_philomena() { + html( + "text text\n---", + "
text text
\n
\n", + ); +} + +#[test] +fn greentext_preserved() { + html( + ">implying\n>>implying", + "
>implying
\n>>implying
\n", + ); +} + +#[test] +fn separate_quotes_on_line_end() { + html( + "> 1\n>\n> 2", + "
\n
1
\n
\n
>
\n
\n
2
\n
\n" + ); +} + +#[test] +fn unnest_quotes_on_line_end() { + html( + "> 1\n> > 2\n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn unnest_quotes_on_line_end_commonmark() { + html( + "> 1\n> > 2\n> \n> 1", + "
\n
1
\n
\n
2
\n
\n
1
\n
\n", + ); +} + +#[test] +fn philomena_images() { + html( + "![full](http://example.com/image.png)", + "
\"full\"
\n", + ); +} + +#[test] +fn no_empty_link() { + html_opts_i( + "[](https://example.com/evil.domain.for.seo.spam)", + "
[](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); + + html_opts_i( + "[ ](https://example.com/evil.domain.for.seo.spam)", + "
[ ](https://example.com/evil.domain.for.seo.spam)
\n", + |opts| opts.extension.autolink = false, + ); +} + +#[test] +fn empty_image_allowed() { + html( + "![ ](https://example.com/evil.domain.for.seo.spam)", + "
\"
\n", + ); +} + +#[test] +fn image_inside_link_allowed() { + html( + "[![](https://example.com/image.png)](https://example.com/)", + "
\"\"
\n", + ); +} + +#[test] +fn image_mention() { + html_opts_i( + "hello world >>1234p >>1337", + "
hello world
p
>>1337
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn image_mention_line_start() { + html_opts_i( + ">>1234p", + "
p
\n", + |opts| { + let mut replacements = HashMap::new(); + replacements.insert("1234p".to_string(), "
p
".to_string()); + + opts.extension.replacements = Some(replacements); + }, + ); +} + +#[test] +fn auto_relative_links() { + let domains: Vec = vec!["example.com".into()]; + let f = Arc::new(move |url: &str| domains::relativize_careful(&domains, url)); + + html_opts_i( + "[some link text](https://example.com/some/path)", + "
some link text
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path", + "
https://example.com/some/path
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "[some link text](https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345)", + "
some link text
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); + + html_opts_i( + "https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345", + "
https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345
\n", + |opts| { + opts.extension.link_url_rewriter = Some(f.clone()); + }, + ); +} From 7ccfc95940ac344af4f43000991b6584a34a18b9 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 11:08:08 -0500 Subject: [PATCH 08/33] Rearrange mix format rule --- .github/workflows/elixir.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 38b201d4..066732f5 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -31,12 +31,12 @@ jobs: - run: docker compose pull - run: docker compose build - - name: mix format - run: docker compose run app mix format --check-formatted - - name: Build and test run: docker compose run app run-test + - name: mix format + run: docker compose run app mix format --check-formatted + - name: Security lint run: | docker compose run app mix sobelow --config From 4f144aff131ddb12edc38a9eb1efed3f89d01af2 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 13:22:51 -0500 Subject: [PATCH 09/33] Upgrade log mutes for new compose syntax --- docker-compose.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5b102408..a1233418 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,16 +64,14 @@ services: - POSTGRES_PASSWORD=postgres volumes: - postgres_data:/var/lib/postgresql/data - logging: - driver: "none" + attach: false opensearch: image: opensearchproject/opensearch:2.16.0 volumes: - opensearch_data:/usr/share/opensearch/data - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml - logging: - driver: "none" + attach: false ulimits: nofile: soft: 65536 @@ -81,8 +79,7 @@ services: valkey: image: valkey/valkey:8.0-alpine - logging: - driver: "none" + attach: false files: image: andrewgaul/s3proxy:sha-4976e17 @@ -90,6 +87,7 @@ services: - JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3 volumes: - .:/srv/philomena + attach: false web: build: @@ -106,8 +104,7 @@ services: environment: - AWS_ACCESS_KEY_ID=local-identity - AWS_SECRET_ACCESS_KEY=local-credential - logging: - driver: "none" + attach: false depends_on: - app ports: From b5bf498fd53410350a77d62f80806233072dff41 Mon Sep 17 00:00:00 2001 From: Konstantin Kolomoets Date: Sun, 10 Nov 2024 02:27:15 +0400 Subject: [PATCH 10/33] Display pagination on the bottom of the Commissions page --- .../templates/commission/_directory_results.html.slime | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/philomena_web/templates/commission/_directory_results.html.slime b/lib/philomena_web/templates/commission/_directory_results.html.slime index a5e7ed0d..3e8b1aeb 100644 --- a/lib/philomena_web/templates/commission/_directory_results.html.slime +++ b/lib/philomena_web/templates/commission/_directory_results.html.slime @@ -62,3 +62,6 @@ elixir: - true -> p We couldn't find any commission listings to display. Sorry! + + .block__header.page__header + .page__pagination = pagination From 50ceeab6f83acf9f60e8aaa2623dc08e6b69f373 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 21:40:15 -0500 Subject: [PATCH 11/33] Ensure local autocomplete works with search form box --- lib/philomena_web/templates/search/_form.html.slime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/philomena_web/templates/search/_form.html.slime b/lib/philomena_web/templates/search/_form.html.slime index efc065af..7d0f080a 100644 --- a/lib/philomena_web/templates/search/_form.html.slime +++ b/lib/philomena_web/templates/search/_form.html.slime @@ -1,7 +1,7 @@ h1 Search = form_for :search, ~p"/search", [id: "searchform", method: "get", class: "js-search-form", enforce_utf8: false], fn f -> - = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"] + = text_input f, :q, class: "input input--wide js-search-field", placeholder: "Search terms are chained with commas", autocapitalize: "none", name: "q", value: @conn.params["q"], data: [ac: "true", ac_min_length: 3, ac_mode: "search"] .block .block__header.flex From aed938bc1764c9d77e8cdff59940b3a9735763a0 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 9 Nov 2024 22:54:09 -0500 Subject: [PATCH 12/33] Convert search help box to TypeScript --- assets/js/__tests__/search.spec.ts | 99 ++++++++++++++++++++++++++++++ assets/js/search.js | 45 -------------- assets/js/search.ts | 85 +++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 45 deletions(-) create mode 100644 assets/js/__tests__/search.spec.ts delete mode 100644 assets/js/search.js create mode 100644 assets/js/search.ts diff --git a/assets/js/__tests__/search.spec.ts b/assets/js/__tests__/search.spec.ts new file mode 100644 index 00000000..abc033b5 --- /dev/null +++ b/assets/js/__tests__/search.spec.ts @@ -0,0 +1,99 @@ +import { $ } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { setupSearch } from '../search'; +import { setupTagListener } from '../tagsinput'; + +const formData = `
+ + NOT + Numeric ID + My favorites + + +
`; + +describe('Search form help', () => { + beforeAll(() => { + setupSearch(); + setupTagListener(); + }); + + let input: HTMLInputElement; + let prependAnchor: HTMLAnchorElement; + let idAnchor: HTMLAnchorElement; + let favesAnchor: HTMLAnchorElement; + let helpNumeric: HTMLDivElement; + let subjectSpan: HTMLElement; + + beforeEach(() => { + document.body.innerHTML = formData; + + input = assertNotNull($('input')); + prependAnchor = assertNotNull($('a[data-search-prepend]')); + idAnchor = assertNotNull($('a[data-search-add="id.lte:10"]')); + favesAnchor = assertNotNull($('a[data-search-add="my:faves"]')); + helpNumeric = assertNotNull($('[data-search-help="numeric"]')); + subjectSpan = assertNotNull($('span', helpNumeric)); + }); + + it('should add text to input field', () => { + idAnchor.click(); + expect(input.value).toBe('id.lte:10'); + + favesAnchor.click(); + expect(input.value).toBe('id.lte:10, my:faves'); + }); + + it('should focus and select text in input field when requested', () => { + idAnchor.click(); + expect(input).toHaveFocus(); + expect(input.selectionStart).toBe(7); + expect(input.selectionEnd).toBe(9); + }); + + it('should highlight subject name when requested', () => { + expect(helpNumeric).toHaveClass('hidden'); + idAnchor.click(); + expect(helpNumeric).not.toHaveClass('hidden'); + expect(subjectSpan).toHaveTextContent('Numeric ID'); + }); + + it('should not focus and select text in input field when unavailable', () => { + favesAnchor.click(); + expect(input).not.toHaveFocus(); + expect(input.selectionStart).toBe(8); + expect(input.selectionEnd).toBe(8); + }); + + it('should not highlight subject name when unavailable', () => { + favesAnchor.click(); + expect(helpNumeric).toHaveClass('hidden'); + }); + + it('should prepend to empty input', () => { + prependAnchor.click(); + expect(input.value).toBe('-'); + }); + + it('should prepend to single input', () => { + input.value = 'a'; + prependAnchor.click(); + expect(input.value).toBe('-a'); + }); + + it('should prepend to comma-separated input', () => { + input.value = 'a,b'; + prependAnchor.click(); + expect(input.value).toBe('a,-b'); + }); + + it('should prepend to comma and space-separated input', () => { + input.value = 'a, b'; + prependAnchor.click(); + expect(input.value).toBe('a, -b'); + }); +}); diff --git a/assets/js/search.js b/assets/js/search.js deleted file mode 100644 index 50733fd9..00000000 --- a/assets/js/search.js +++ /dev/null @@ -1,45 +0,0 @@ -import { $, $$ } from './utils/dom'; -import { addTag } from './tagsinput'; - -function showHelp(subject, type) { - $$('[data-search-help]').forEach(helpBox => { - if (helpBox.getAttribute('data-search-help') === type) { - $('.js-search-help-subject', helpBox).textContent = subject; - helpBox.classList.remove('hidden'); - } else { - helpBox.classList.add('hidden'); - } - }); -} - -function prependToLast(field, value) { - const separatorIndex = field.value.lastIndexOf(','); - const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1; - field.value = - field.value.slice(0, separatorIndex + advanceBy) + value + field.value.slice(separatorIndex + advanceBy); -} - -function selectLast(field, characterCount) { - field.focus(); - - field.selectionStart = field.value.length - characterCount; - field.selectionEnd = field.value.length; -} - -function executeFormHelper(e) { - const searchField = $('.js-search-field'); - const attr = name => e.target.getAttribute(name); - - attr('data-search-add') && addTag(searchField, attr('data-search-add')); - attr('data-search-show-help') && showHelp(e.target.textContent, attr('data-search-show-help')); - attr('data-search-select-last') && selectLast(searchField, parseInt(attr('data-search-select-last'), 10)); - attr('data-search-prepend') && prependToLast(searchField, attr('data-search-prepend')); -} - -function setupSearch() { - const form = $('.js-search-form'); - - form && form.addEventListener('click', executeFormHelper); -} - -export { setupSearch }; diff --git a/assets/js/search.ts b/assets/js/search.ts new file mode 100644 index 00000000..eff8d98a --- /dev/null +++ b/assets/js/search.ts @@ -0,0 +1,85 @@ +import { assertNotNull, assertNotUndefined } from './utils/assert'; +import { $, $$, showEl, hideEl } from './utils/dom'; +import { delegate, leftClick } from './utils/events'; +import { addTag } from './tagsinput'; + +function focusAndSelectLast(field: HTMLInputElement, characterCount: number) { + field.focus(); + field.selectionStart = field.value.length - characterCount; + field.selectionEnd = field.value.length; +} + +function prependToLast(field: HTMLInputElement, value: string) { + // Find the last comma in the input and advance past it + const separatorIndex = field.value.lastIndexOf(','); + const advanceBy = field.value[separatorIndex + 1] === ' ' ? 2 : 1; + + // Insert the value string at the new location + field.value = [ + field.value.slice(0, separatorIndex + advanceBy), + value, + field.value.slice(separatorIndex + advanceBy), + ].join(''); +} + +function getAssociatedData(target: HTMLElement) { + const form = assertNotNull(target.closest('form')); + const input = assertNotNull($('.js-search-field', form)); + const helpBoxes = $$('[data-search-help]', form); + + return { input, helpBoxes }; +} + +function showHelp(helpBoxes: HTMLDivElement[], typeName: string, subject: string) { + for (const helpBox of helpBoxes) { + // Get the subject name span + const subjectName = assertNotNull($('.js-search-help-subject', helpBox)); + + // Take the appropriate action for this help box + if (helpBox.dataset.searchHelp === typeName) { + subjectName.textContent = subject; + showEl(helpBox); + } else { + hideEl(helpBox); + } + } +} + +function onSearchAdd(_event: Event, target: HTMLAnchorElement) { + // Load form + const { input, helpBoxes } = getAssociatedData(target); + + // Get data for this link + const addValue = assertNotUndefined(target.dataset.searchAdd); + const showHelpValue = assertNotUndefined(target.dataset.searchShowHelp); + const selectLastValue = target.dataset.searchSelectLast; + + // Add the tag + addTag(input, addValue); + + // Show associated help, if available + showHelp(helpBoxes, showHelpValue, assertNotNull(target.textContent)); + + // Select last characters, if requested + if (selectLastValue) { + focusAndSelectLast(input, Number(selectLastValue)); + } +} + +function onSearchPrepend(_event: Event, target: HTMLAnchorElement) { + // Load form + const { input } = getAssociatedData(target); + + // Get data for this link + const prependValue = assertNotUndefined(target.dataset.searchPrepend); + + // Prepend + prependToLast(input, prependValue); +} + +export function setupSearch() { + delegate(document, 'click', { + 'form.js-search-form a[data-search-add][data-search-show-help]': leftClick(onSearchAdd), + 'form.js-search-form a[data-search-prepend]': leftClick(onSearchPrepend), + }); +} From a5bec50ba7d559ac1fd449f3961d9fb771033115 Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 11 Nov 2024 10:22:54 -0500 Subject: [PATCH 13/33] Convert tagsinput to TypeScript --- assets/js/__tests__/tagsinput.spec.ts | 188 +++++++++++++++++++++++ assets/js/{tagsinput.js => tagsinput.ts} | 64 ++++---- assets/test/vitest-setup.ts | 2 + assets/types/booru-object.d.ts | 8 + assets/types/tags.ts | 20 +++ 5 files changed, 253 insertions(+), 29 deletions(-) create mode 100644 assets/js/__tests__/tagsinput.spec.ts rename assets/js/{tagsinput.js => tagsinput.ts} (63%) create mode 100644 assets/types/tags.ts diff --git a/assets/js/__tests__/tagsinput.spec.ts b/assets/js/__tests__/tagsinput.spec.ts new file mode 100644 index 00000000..e32b2dfc --- /dev/null +++ b/assets/js/__tests__/tagsinput.spec.ts @@ -0,0 +1,188 @@ +import { $, $$, hideEl } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { TermSuggestion } from '../utils/suggestions'; +import { setupTagsInput, addTag, reloadTagsInput } from '../tagsinput'; + +const formData = `
+
+ +
+ +
+
+ + + +
`; + +describe('Fancy tags input', () => { + let form: HTMLFormElement; + let tagBlock: HTMLDivElement; + let plainInput: HTMLTextAreaElement; + let fancyInput: HTMLDivElement; + let fancyText: HTMLInputElement; + let fancyShowButton: HTMLButtonElement; + let plainShowButton: HTMLButtonElement; + + beforeEach(() => { + window.booru.fancyTagUpload = true; + window.booru.fancyTagEdit = true; + document.body.innerHTML = formData; + + form = assertNotNull($('.tags-form')); + tagBlock = assertNotNull($('.js-tag-block')); + plainInput = assertNotNull($('.js-taginput-plain')); + fancyInput = assertNotNull($('.js-taginput-fancy')); + fancyText = assertNotNull($('.js-taginput-input')); + fancyShowButton = assertNotNull($('.js-taginput-show')); + plainShowButton = assertNotNull($('.js-taginput-hide')); + + // prevent these from submitting the form + fancyShowButton.addEventListener('click', e => e.preventDefault()); + plainShowButton.addEventListener('click', e => e.preventDefault()); + }); + + for (let i = 0; i < 4; i++) { + const type = (i & 2) === 0 ? 'upload' : 'edit'; + const name = (i & 2) === 0 ? 'fancyTagUpload' : 'fancyTagEdit'; + const value = (i & 1) === 0; + + // eslint-disable-next-line no-loop-func + it(`should imply ${name}:${value} <-> ${type}:${value} on setup`, () => { + window.booru.fancyTagEdit = false; + window.booru.fancyTagUpload = false; + window.booru[name] = value; + + plainInput.value = 'a, b'; + tagBlock.classList.remove('fancy-tag-edit', 'fancy-tag-upload'); + tagBlock.classList.add(`fancy-tag-${type}`); + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + expect($$('span.tag', fancyInput)).toHaveLength(value ? 2 : 0); + }); + } + + it('should move tags from the plain to the fancy editor when the fancy editor is shown', () => { + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + plainInput.value = 'a, b'; + fancyShowButton.click(); + expect($$('span.tag', fancyInput)).toHaveLength(2); + }); + + it('should move tags from the plain to the fancy editor on reload event', () => { + expect($$('span.tag', fancyInput)).toHaveLength(0); + + setupTagsInput(tagBlock); + plainInput.value = 'a, b'; + reloadTagsInput(plainInput); + expect($$('span.tag', fancyInput)).toHaveLength(2); + }); + + it('should respond to addtag events', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should not respond to addtag events if the container is hidden', () => { + setupTagsInput(tagBlock); + hideEl(fancyInput); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should respond to autocomplete events', () => { + setupTagsInput(tagBlock); + fancyText.dispatchEvent(new CustomEvent('autocomplete', { detail: { value: 'a', label: 'a' } })); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should allow removing previously added tags by clicking them', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + assertNotNull($('span.tag a', fancyInput)).click(); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should allow removing previously added tags by adding one with a minus sign prepended', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + addTag(plainInput, '-a'); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should disallow adding empty tags', () => { + setupTagsInput(tagBlock); + addTag(plainInput, ''); + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('should disallow adding existing tags', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + addTag(plainInput, 'a'); + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('should submit the form on ctrl+enter', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 13, ctrlKey: true, bubbles: true }); + + return new Promise(resolve => { + form.addEventListener('submit', e => { + e.preventDefault(); + resolve(); + }); + + fancyText.dispatchEvent(ev); + expect(ev.defaultPrevented).toBe(true); + }); + }); + + it('does nothing when backspacing on empty input and there are no tags', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true }); + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(0); + }); + + it('erases the last added tag when backspacing on empty input', () => { + setupTagsInput(tagBlock); + addTag(plainInput, 'a'); + addTag(plainInput, 'b'); + + const ev = new KeyboardEvent('keydown', { keyCode: 8, bubbles: true }); + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + }); + + it('adds new tag when comma is pressed', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 188, bubbles: true }); + fancyText.value = 'a'; + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + expect(fancyText.value).toBe(''); + }); + + it('adds new tag when enter is pressed', () => { + setupTagsInput(tagBlock); + + const ev = new KeyboardEvent('keydown', { keyCode: 13, bubbles: true }); + fancyText.value = 'a'; + fancyText.dispatchEvent(ev); + + expect($$('span.tag', fancyInput)).toHaveLength(1); + expect(fancyText.value).toBe(''); + }); +}); diff --git a/assets/js/tagsinput.js b/assets/js/tagsinput.ts similarity index 63% rename from assets/js/tagsinput.js rename to assets/js/tagsinput.ts index bc05b3d7..377db60d 100644 --- a/assets/js/tagsinput.js +++ b/assets/js/tagsinput.ts @@ -2,14 +2,20 @@ * Fancy tag editor. */ +import { assertNotNull, assertType } from './utils/assert'; import { $, $$, clearEl, removeEl, showEl, hideEl, escapeCss, escapeHtml } from './utils/dom'; +import { TermSuggestion } from './utils/suggestions'; -function setupTagsInput(tagBlock) { - const [textarea, container] = $$('.js-taginput', tagBlock); - const setup = $('.js-tag-block ~ button', tagBlock.parentNode); - const inputField = $('input', container); +export function setupTagsInput(tagBlock: HTMLDivElement) { + const form = assertNotNull(tagBlock.closest('form')); + const textarea = assertNotNull($('.js-taginput-plain', tagBlock)); + const container = assertNotNull($('.js-taginput-fancy')); + const parentField = assertNotNull(tagBlock.parentElement); + const setup = assertNotNull($('.js-tag-block ~ button', parentField)); + const inputField = assertNotNull($('input', container)); + const submitButton = assertNotNull($('[type="submit"]', form)); - let tags = []; + let tags: string[] = []; // Load in the current tag set from the textarea setup.addEventListener('click', importTags); @@ -27,7 +33,7 @@ function setupTagsInput(tagBlock) { inputField.addEventListener('keydown', handleKeyEvent); // Respond to autocomplete form clicks - inputField.addEventListener('autocomplete', handleAutocomplete); + inputField.addEventListener('autocomplete', handleAutocomplete as EventListener); // Respond to Ctrl+Enter shortcut tagBlock.addEventListener('keydown', handleCtrlEnter); @@ -35,19 +41,19 @@ function setupTagsInput(tagBlock) { // TODO: Cleanup this bug fix // Switch to fancy tagging if user settings want it if (fancyEditorRequested(tagBlock)) { - showEl($$('.js-taginput-fancy')); - showEl($$('.js-taginput-hide')); - hideEl($$('.js-taginput-plain')); - hideEl($$('.js-taginput-show')); + showEl($$('.js-taginput-fancy')); + showEl($$('.js-taginput-hide')); + hideEl($$('.js-taginput-plain')); + hideEl($$('.js-taginput-show')); importTags(); } - function handleAutocomplete(event) { + function handleAutocomplete(event: CustomEvent) { insertTag(event.detail.value); inputField.focus(); } - function handleAddTag(event) { + function handleAddTag(event: AddtagEvent) { // Ignore if not in tag edit mode if (container.classList.contains('hidden')) return; @@ -55,14 +61,16 @@ function setupTagsInput(tagBlock) { event.stopPropagation(); } - function handleTagClear(event) { - if (event.target.dataset.tagName) { + function handleTagClear(event: Event) { + const target = assertType(event.target, HTMLElement); + + if (target.dataset.tagName) { event.preventDefault(); - removeTag(event.target.dataset.tagName, event.target.parentNode); + removeTag(target.dataset.tagName, assertNotNull(target.parentElement)); } } - function handleKeyEvent(event) { + function handleKeyEvent(event: KeyboardEvent) { const { keyCode, ctrlKey, shiftKey } = event; // allow form submission with ctrl+enter if no text was typed @@ -73,7 +81,7 @@ function setupTagsInput(tagBlock) { // backspace on a blank input field if (keyCode === 8 && inputField.value === '') { event.preventDefault(); - const erased = $('.tag:last-of-type', container); + const erased = $('.tag:last-of-type', container); if (erased) removeTag(tags[tags.length - 1], erased); } @@ -86,14 +94,14 @@ function setupTagsInput(tagBlock) { } } - function handleCtrlEnter(event) { + function handleCtrlEnter(event: KeyboardEvent) { const { keyCode, ctrlKey } = event; if (keyCode !== 13 || !ctrlKey) return; - $('[type="submit"]', tagBlock.closest('form')).click(); + submitButton.click(); } - function insertTag(name) { + function insertTag(name: string) { name = name.trim(); // eslint-disable-line no-param-reassign // Add if not degenerate or already present @@ -102,9 +110,9 @@ function setupTagsInput(tagBlock) { // Remove instead if the tag name starts with a minus if (name[0] === '-') { name = name.slice(1); // eslint-disable-line no-param-reassign - const tagLink = $(`[data-tag-name="${escapeCss(name)}"]`, container); + const tagLink = assertNotNull($(`[data-tag-name="${escapeCss(name)}"]`, container)); - return removeTag(name, tagLink.parentNode); + return removeTag(name, assertNotNull(tagLink.parentElement)); } tags.push(name); @@ -116,7 +124,7 @@ function setupTagsInput(tagBlock) { inputField.value = ''; } - function removeTag(name, element) { + function removeTag(name: string, element: HTMLElement) { removeEl(element); // Remove the tag from the list @@ -134,7 +142,7 @@ function setupTagsInput(tagBlock) { } } -function fancyEditorRequested(tagBlock) { +function fancyEditorRequested(tagBlock: HTMLDivElement) { // Check whether the user made the fancy editor the default for each type of tag block. return ( (window.booru.fancyTagUpload && tagBlock.classList.contains('fancy-tag-upload')) || @@ -142,19 +150,17 @@ function fancyEditorRequested(tagBlock) { ); } -function setupTagListener() { +export function setupTagListener() { document.addEventListener('addtag', event => { if (event.target.value) event.target.value += ', '; event.target.value += event.detail.name; }); } -function addTag(textarea, name) { +export function addTag(textarea: HTMLInputElement | HTMLTextAreaElement, name: string) { textarea.dispatchEvent(new CustomEvent('addtag', { detail: { name }, bubbles: true })); } -function reloadTagsInput(textarea) { +export function reloadTagsInput(textarea: HTMLInputElement | HTMLTextAreaElement) { textarea.dispatchEvent(new CustomEvent('reload')); } - -export { setupTagsInput, setupTagListener, addTag, reloadTagsInput }; diff --git a/assets/test/vitest-setup.ts b/assets/test/vitest-setup.ts index 7c0d2d62..ac349f86 100644 --- a/assets/test/vitest-setup.ts +++ b/assets/test/vitest-setup.ts @@ -7,6 +7,8 @@ import { fireEvent } from '@testing-library/dom'; window.booru = { timeAgo: () => {}, csrfToken: 'mockCsrfToken', + fancyTagEdit: true, + fancyTagUpload: true, hiddenTag: '/mock-tagblocked.svg', hiddenTagList: [], hideStaffTools: 'true', diff --git a/assets/types/booru-object.d.ts b/assets/types/booru-object.d.ts index 4154385c..8b04cced 100644 --- a/assets/types/booru-object.d.ts +++ b/assets/types/booru-object.d.ts @@ -73,6 +73,14 @@ interface BooruObject { * List of image IDs in the current gallery. */ galleryImages?: number[]; + /** + * Fancy tag setting for uploading images. + */ + fancyTagUpload: boolean; + /** + * Fancy tag setting for editing images. + */ + fancyTagEdit: boolean; } declare global { diff --git a/assets/types/tags.ts b/assets/types/tags.ts new file mode 100644 index 00000000..b132dbf0 --- /dev/null +++ b/assets/types/tags.ts @@ -0,0 +1,20 @@ +export {}; + +declare global { + interface Addtag { + name: string; + } + + interface AddtagEvent extends CustomEvent { + target: HTMLInputElement | HTMLTextAreaElement; + } + + interface ReloadEvent extends CustomEvent { + target: HTMLInputElement | HTMLTextAreaElement; + } + + interface GlobalEventHandlersEventMap { + addtag: AddtagEvent; + reload: ReloadEvent; + } +} From 0523c0f400a502aeb6597621abf9588eba3e2933 Mon Sep 17 00:00:00 2001 From: KoloMl Date: Thu, 14 Nov 2024 22:33:10 +0400 Subject: [PATCH 14/33] Fixed search fields resetting to the last term instead of the query --- assets/js/autocomplete.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/js/autocomplete.ts b/assets/js/autocomplete.ts index 489392c3..6a8c560e 100644 --- a/assets/js/autocomplete.ts +++ b/assets/js/autocomplete.ts @@ -25,6 +25,7 @@ function restoreOriginalValue() { if (isSearchField(inputField) && originalQuery) { inputField.value = originalQuery; + return; } if (originalTerm) { From 8e742b2549bbd3e2b68d79dabfd8133627848c0d Mon Sep 17 00:00:00 2001 From: KoloMl Date: Thu, 14 Nov 2024 22:43:43 +0400 Subject: [PATCH 15/33] Move selection to end of the selected term instead of the end of query --- assets/js/autocomplete.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/assets/js/autocomplete.ts b/assets/js/autocomplete.ts index 6a8c560e..61340b48 100644 --- a/assets/js/autocomplete.ts +++ b/assets/js/autocomplete.ts @@ -25,6 +25,13 @@ function restoreOriginalValue() { if (isSearchField(inputField) && originalQuery) { inputField.value = originalQuery; + + if (selectedTerm) { + const [, selectedTermEnd] = selectedTerm[0]; + + inputField.setSelectionRange(selectedTermEnd, selectedTermEnd); + } + return; } From c06033aa1004bcbd792a651eb1d43a63dc114499 Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 11 Nov 2024 13:43:06 -0500 Subject: [PATCH 16/33] Convert quicktag to TypeScript --- assets/js/__tests__/quick-tag.spec.ts | 159 ++++++++++++++++++++++++++ assets/js/quick-tag.js | 117 ------------------- assets/js/quick-tag.ts | 124 ++++++++++++++++++++ 3 files changed, 283 insertions(+), 117 deletions(-) create mode 100644 assets/js/__tests__/quick-tag.spec.ts delete mode 100644 assets/js/quick-tag.js create mode 100644 assets/js/quick-tag.ts diff --git a/assets/js/__tests__/quick-tag.spec.ts b/assets/js/__tests__/quick-tag.spec.ts new file mode 100644 index 00000000..6b5d1ecd --- /dev/null +++ b/assets/js/__tests__/quick-tag.spec.ts @@ -0,0 +1,159 @@ +import { $, $$ } from '../utils/dom'; +import { assertNotNull } from '../utils/assert'; +import { setupQuickTag } from '../quick-tag'; +import { fetchMock } from '../../test/fetch-mock.ts'; +import { waitFor } from '@testing-library/dom'; + +const quickTagData = `
+ Tag + + + +
+
+
+
+
+
+
+
+
`; + +describe('Batch tagging', () => { + let tagButton: HTMLAnchorElement; + let abortButton: HTMLAnchorElement; + let submitButton: HTMLAnchorElement; + let toggleAllButton: HTMLAnchorElement; + let mediaBoxes: HTMLDivElement[]; + + beforeEach(() => { + localStorage.clear(); + document.body.innerHTML = quickTagData; + + tagButton = assertNotNull($('.js-quick-tag')); + abortButton = assertNotNull($('.js-quick-tag--abort')); + submitButton = assertNotNull($('.js-quick-tag--submit')); + toggleAllButton = assertNotNull($('.js-quick-tag--all')); + mediaBoxes = $$('.media-box'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should prompt the user on click', () => { + const spy = vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).toContain('hidden'); + expect(abortButton.classList).not.toContain('hidden'); + expect(submitButton.classList).not.toContain('hidden'); + expect(toggleAllButton.classList).not.toContain('hidden'); + }); + + it('should not modify media boxes before entry', () => { + mediaBoxes[0].click(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + + it('should restore the list of tagged images on reload', () => { + // TODO: this is less than ideal, because it depends on the internal + // implementation of the quick-tag file. But we can't reload the page + // with jsdom. + localStorage.setItem('quickTagQueue', JSON.stringify(['0', '1'])); + localStorage.setItem('quickTagName', JSON.stringify('a')); + + setupQuickTag(); + expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected'); + }); + + describe('after entry', () => { + beforeEach(() => { + vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + }); + + it('should abort the tagging process if accepted', () => { + const spy = vi.spyOn(window, 'confirm').mockImplementation(() => true); + abortButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).not.toContain('hidden'); + expect(abortButton.classList).toContain('hidden'); + expect(submitButton.classList).toContain('hidden'); + expect(toggleAllButton.classList).toContain('hidden'); + }); + + it('should not abort the tagging process if rejected', () => { + const spy = vi.spyOn(window, 'confirm').mockImplementation(() => false); + abortButton.click(); + + expect(spy).toHaveBeenCalledOnce(); + expect(tagButton.classList).toContain('hidden'); + expect(abortButton.classList).not.toContain('hidden'); + expect(submitButton.classList).not.toContain('hidden'); + expect(toggleAllButton.classList).not.toContain('hidden'); + }); + + it('should toggle media box state on click', () => { + mediaBoxes[0].click(); + expect(mediaBoxes[0].firstElementChild).toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + + it('should toggle all media box states', () => { + mediaBoxes[0].click(); + toggleAllButton.click(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).toHaveClass('media-box__header--selected'); + }); + }); + + describe('for submission', () => { + beforeAll(() => { + fetchMock.enableMocks(); + }); + + afterAll(() => { + fetchMock.disableMocks(); + }); + + beforeEach(() => { + vi.spyOn(window, 'prompt').mockImplementation(() => 'a'); + tagButton.click(); + + fetchMock.resetMocks(); + mediaBoxes[0].click(); + mediaBoxes[1].click(); + }); + + it('should return to normal state on successful submission', () => { + fetchMock.mockResponse('{"failed":[]}'); + submitButton.click(); + + expect(fetch).toHaveBeenCalledOnce(); + + return waitFor(() => { + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + }); + + it('should show error on failed submission', () => { + fetchMock.mockResponse('{"failed":[0,1]}'); + submitButton.click(); + + const spy = vi.spyOn(window, 'alert').mockImplementation(() => {}); + + expect(fetch).toHaveBeenCalledOnce(); + + return waitFor(() => { + expect(spy).toHaveBeenCalledOnce(); + expect(mediaBoxes[0].firstElementChild).not.toHaveClass('media-box__header--selected'); + expect(mediaBoxes[1].firstElementChild).not.toHaveClass('media-box__header--selected'); + }); + }); + }); +}); diff --git a/assets/js/quick-tag.js b/assets/js/quick-tag.js deleted file mode 100644 index 4457784a..00000000 --- a/assets/js/quick-tag.js +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Quick Tag - */ - -import store from './utils/store'; -import { $, $$, toggleEl, onLeftClick } from './utils/dom'; -import { fetchJson, handleError } from './utils/requests'; - -const imageQueueStorage = 'quickTagQueue'; -const currentTagStorage = 'quickTagName'; - -function currentQueue() { - return store.get(imageQueueStorage) || []; -} - -function currentTags() { - return store.get(currentTagStorage) || ''; -} - -function getTagButton() { - return $('.js-quick-tag'); -} - -function setTagButton(text) { - $('.js-quick-tag--submit span').textContent = text; -} - -function toggleActiveState() { - toggleEl($('.js-quick-tag'), $('.js-quick-tag--abort'), $('.js-quick-tag--all'), $('.js-quick-tag--submit')); - - setTagButton(`Submit (${currentTags()})`); - - $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected')); - $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected')); - currentQueue().forEach(id => - $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')), - ); -} - -function activate() { - store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:')); - - if (currentTags()) toggleActiveState(); -} - -function reset() { - store.remove(currentTagStorage); - store.remove(imageQueueStorage); - - toggleActiveState(); -} - -function promptReset() { - if (window.confirm('Are you sure you want to abort batch tagging?')) { - reset(); - } -} - -function submit() { - setTagButton(`Wait... (${currentTags()})`); - - fetchJson('PUT', '/admin/batch/tags', { - tags: currentTags(), - image_ids: currentQueue(), - }) - .then(handleError) - .then(r => r.json()) - .then(data => { - if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`); - - reset(); - }); -} - -function modifyImageQueue(mediaBox) { - if (currentTags()) { - const imageId = mediaBox.dataset.imageId; - const queue = currentQueue(); - const isSelected = queue.includes(imageId); - - isSelected ? queue.splice(queue.indexOf(imageId), 1) : queue.push(imageId); - - $$(`.media-box__header[data-image-id="${imageId}"]`).forEach(el => - el.classList.toggle('media-box__header--selected'), - ); - - store.set(imageQueueStorage, queue); - } -} - -function toggleAllImages() { - $$('#imagelist-container .media-box').forEach(modifyImageQueue); -} - -function clickHandler(event) { - const targets = { - '.js-quick-tag': activate, - '.js-quick-tag--abort': promptReset, - '.js-quick-tag--submit': submit, - '.js-quick-tag--all': toggleAllImages, - '.media-box': modifyImageQueue, - }; - - for (const target in targets) { - if (event.target && event.target.closest(target)) { - targets[target](event.target.closest(target)); - currentTags() && event.preventDefault(); - } - } -} - -function setupQuickTag() { - if (getTagButton() && currentTags()) toggleActiveState(); - if (getTagButton()) onLeftClick(clickHandler); -} - -export { setupQuickTag }; diff --git a/assets/js/quick-tag.ts b/assets/js/quick-tag.ts new file mode 100644 index 00000000..3d462cb2 --- /dev/null +++ b/assets/js/quick-tag.ts @@ -0,0 +1,124 @@ +/** + * Quick Tag + */ + +import store from './utils/store'; +import { assertNotNull, assertNotUndefined } from './utils/assert'; +import { $, $$, toggleEl } from './utils/dom'; +import { fetchJson, handleError } from './utils/requests'; +import { delegate, leftClick } from './utils/events'; + +const imageQueueStorage = 'quickTagQueue'; +const currentTagStorage = 'quickTagName'; + +function currentQueue(): string[] { + return store.get(imageQueueStorage) || []; +} + +function currentTags(): string { + return store.get(currentTagStorage) || ''; +} + +function setTagButton(text: string) { + assertNotNull($('.js-quick-tag--submit span')).textContent = text; +} + +function toggleActiveState() { + toggleEl($$('.js-quick-tag,.js-quick-tag--abort,.js-quick-tag--all,.js-quick-tag--submit')); + + setTagButton(`Submit (${currentTags()})`); + + $$('.media-box__header').forEach(el => el.classList.toggle('media-box__header--unselected')); + $$('.media-box__header').forEach(el => el.classList.remove('media-box__header--selected')); + + currentQueue().forEach(id => + $$(`.media-box__header[data-image-id="${id}"]`).forEach(el => el.classList.add('media-box__header--selected')), + ); +} + +function activate(event: Event) { + event.preventDefault(); + + store.set(currentTagStorage, window.prompt('A comma-delimited list of tags you want to add:')); + + if (currentTags()) { + toggleActiveState(); + } +} + +function reset() { + store.remove(currentTagStorage); + store.remove(imageQueueStorage); + + toggleActiveState(); +} + +function promptReset(event: Event) { + event.preventDefault(); + + if (window.confirm('Are you sure you want to abort batch tagging?')) { + reset(); + } +} + +function submit(event: Event) { + event.preventDefault(); + + setTagButton(`Wait... (${currentTags()})`); + + fetchJson('PUT', '/admin/batch/tags', { + tags: currentTags(), + image_ids: currentQueue(), + }) + .then(handleError) + .then(r => r.json()) + .then(data => { + if (data.failed.length) window.alert(`Failed to add tags to the images with these IDs: ${data.failed}`); + + reset(); + }); +} + +function modifyImageQueue(event: Event, mediaBox: HTMLDivElement) { + if (!currentTags()) { + return; + } + + const imageId = assertNotUndefined(mediaBox.dataset.imageId); + const queue = currentQueue(); + const isSelected = queue.includes(imageId); + + if (isSelected) { + queue.splice(queue.indexOf(imageId), 1); + } else { + queue.push(imageId); + } + + for (const boxHeader of $$(`.media-box__header[data-image-id="${imageId}"]`)) { + boxHeader.classList.toggle('media-box__header--selected'); + } + + store.set(imageQueueStorage, queue); + event.preventDefault(); +} + +function toggleAllImages(event: Event, _target: Element) { + for (const mediaBox of $$('#imagelist-container .media-box')) { + modifyImageQueue(event, mediaBox); + } +} + +delegate(document, 'click', { + '.js-quick-tag': leftClick(activate), + '.js-quick-tag--abort': leftClick(promptReset), + '.js-quick-tag--submit': leftClick(submit), + '.js-quick-tag--all': leftClick(toggleAllImages), + '.media-box': leftClick(modifyImageQueue), +}); + +export function setupQuickTag() { + const tagButton = $('.js-quick-tag'); + if (tagButton && currentTags()) { + toggleActiveState(); + } +} From 9e35238cab4bc0f8ac91b43f25bc06a3dbe7049d Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 14 Nov 2024 21:53:29 -0500 Subject: [PATCH 17/33] Fix single-source saving bug --- .../controllers/image/source_controller.ex | 10 +++++++++- lib/philomena_web/controllers/image_controller.ex | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/philomena_web/controllers/image/source_controller.ex b/lib/philomena_web/controllers/image/source_controller.ex index 373dcc10..2dbc4f47 100644 --- a/lib/philomena_web/controllers/image/source_controller.ex +++ b/lib/philomena_web/controllers/image/source_controller.ex @@ -4,6 +4,7 @@ defmodule PhilomenaWeb.Image.SourceController do alias Philomena.SourceChanges.SourceChange alias Philomena.UserStatistics alias Philomena.Images.Image + alias Philomena.Images.Source alias Philomena.Images alias Philomena.Repo import Ecto.Query @@ -41,7 +42,9 @@ defmodule PhilomenaWeb.Image.SourceController do PhilomenaWeb.Api.Json.ImageView.render("show.json", %{image: image, interactions: []}) ) - changeset = Images.change_image(image) + changeset = + %{image | sources: sources_for_edit(image.sources)} + |> Images.change_image() source_change_count = SourceChange @@ -74,4 +77,9 @@ defmodule PhilomenaWeb.Image.SourceController do ) end end + + # TODO: this is duplicated in ImageController + defp sources_for_edit(), do: [%Source{}] + defp sources_for_edit([]), do: sources_for_edit() + defp sources_for_edit(sources), do: sources end diff --git a/lib/philomena_web/controllers/image_controller.ex b/lib/philomena_web/controllers/image_controller.ex index 990fcfd3..4088fe56 100644 --- a/lib/philomena_web/controllers/image_controller.ex +++ b/lib/philomena_web/controllers/image_controller.ex @@ -219,6 +219,7 @@ defmodule PhilomenaWeb.ImageController do end end + # TODO: this is duplicated in Image.SourceController defp sources_for_edit(), do: [%Source{}] defp sources_for_edit([]), do: sources_for_edit() defp sources_for_edit(sources), do: sources From 88f58a5474384d92913e6bcab0caccf2b5f2871f Mon Sep 17 00:00:00 2001 From: Yay295 Date: Thu, 14 Nov 2024 23:48:46 -0600 Subject: [PATCH 18/33] handle bluesky "handle" that is a "did" --- lib/philomena_proxy/scrapers/bluesky.ex | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex index 598d1470..4f52a9cc 100644 --- a/lib/philomena_proxy/scrapers/bluesky.ex +++ b/lib/philomena_proxy/scrapers/bluesky.ex @@ -19,10 +19,15 @@ defmodule PhilomenaProxy.Scrapers.Bluesky do def scrape(_uri, url) do [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) - api_url_resolve_handle = - "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" + did = + if String.starts_with?(handle, "did:") do + handle + else + api_url_resolve_handle = + "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" - did = PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did) + PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did) + end api_url_get_posts = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" From 282b8b73228693f3f72e7b85d1e7819baca8d178 Mon Sep 17 00:00:00 2001 From: Liam Date: Fri, 15 Nov 2024 08:55:33 -0500 Subject: [PATCH 19/33] Fix errors in bsky scraper --- lib/philomena_proxy/scrapers/bluesky.ex | 49 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/lib/philomena_proxy/scrapers/bluesky.ex b/lib/philomena_proxy/scrapers/bluesky.ex index 4f52a9cc..e67672b2 100644 --- a/lib/philomena_proxy/scrapers/bluesky.ex +++ b/lib/philomena_proxy/scrapers/bluesky.ex @@ -19,28 +19,25 @@ defmodule PhilomenaProxy.Scrapers.Bluesky do def scrape(_uri, url) do [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) - did = - if String.starts_with?(handle, "did:") do - handle - else - api_url_resolve_handle = - "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" - - PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did) - end + did = fetch_did(handle) api_url_get_posts = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" - post_json = PhilomenaProxy.Http.get(api_url_get_posts) |> json!() |> Map.fetch!(:posts) |> hd + post_json = + api_url_get_posts + |> PhilomenaProxy.Http.get() + |> json!() + |> Map.fetch!("posts") + |> hd() %{ source_url: url, - author_name: post_json["author"]["handle"], + author_name: domain_first_component(post_json["author"]["handle"]), description: post_json["record"]["text"], images: - post_json["embed"]["images"] - |> Enum.map( + Enum.map( + post_json["embed"]["images"], &%{ url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern), camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"]) @@ -49,5 +46,31 @@ defmodule PhilomenaProxy.Scrapers.Bluesky do } end + defp fetch_did(handle) do + case handle do + <<"did:", _rest::binary>> -> + handle + + _ -> + api_url_resolve_handle = + "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}" + + api_url_resolve_handle + |> PhilomenaProxy.Http.get() + |> json!() + |> Map.fetch!("did") + end + end + + defp domain_first_component(domain) do + case String.split(domain, ".") do + [name | _] -> + name + + _ -> + domain + end + end + defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body) end From a7e58f588437a486d697ef62acffdcad29d8e735 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 16 Nov 2024 12:30:22 -0500 Subject: [PATCH 20/33] Scope tag input select to tag block --- assets/js/tagsinput.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/js/tagsinput.ts b/assets/js/tagsinput.ts index 377db60d..74574906 100644 --- a/assets/js/tagsinput.ts +++ b/assets/js/tagsinput.ts @@ -9,7 +9,7 @@ import { TermSuggestion } from './utils/suggestions'; export function setupTagsInput(tagBlock: HTMLDivElement) { const form = assertNotNull(tagBlock.closest('form')); const textarea = assertNotNull($('.js-taginput-plain', tagBlock)); - const container = assertNotNull($('.js-taginput-fancy')); + const container = assertNotNull($('.js-taginput-fancy', tagBlock)); const parentField = assertNotNull(tagBlock.parentElement); const setup = assertNotNull($('.js-tag-block ~ button', parentField)); const inputField = assertNotNull($('input', container)); From 3c4b7179179df1f4cea09ea71056ec96ce6c9604 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 16 Nov 2024 16:00:58 -0500 Subject: [PATCH 21/33] Add lexical to ignore list --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index da2c70b6..0c221544 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,9 @@ npm-debug.log # VS Code .vscode -# ElixirLS +# Language server .elixir_ls +.lexical # Index dumps *.jsonl From a60c7d9e794c71fc5b418a250b2d1dc26fadfb5e Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 17 Nov 2024 16:38:51 -0500 Subject: [PATCH 22/33] Add testcase for subscript-autolink interaction philomena-dev/philomena#315 --- native/philomena/src/tests.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs index 9f1f963f..f57eb0f2 100644 --- a/native/philomena/src/tests.rs +++ b/native/philomena/src/tests.rs @@ -48,6 +48,14 @@ fn subscript() { html("H%2%O\n", "
H2O
\n"); } +#[test] +fn subscript_autolink_interaction() { + html( + "https://example.com/search?q=1%2C2%2C3", + "
https://example.com/search?q=1%2C2%2C3
\n" + ); +} + #[test] fn spoiler() { html( From 30a552badf0f3d2512bdd6d820092acda3797cc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 22:53:36 +0000 Subject: [PATCH 23/33] Bump cross-spawn from 7.0.3 to 7.0.6 in /assets Bumps [cross-spawn](https://github.com/moxystudio/node-cross-spawn) from 7.0.3 to 7.0.6. - [Changelog](https://github.com/moxystudio/node-cross-spawn/blob/master/CHANGELOG.md) - [Commits](https://github.com/moxystudio/node-cross-spawn/compare/v7.0.3...v7.0.6) --- updated-dependencies: - dependency-name: cross-spawn dependency-type: indirect ... Signed-off-by: dependabot[bot] --- assets/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/package-lock.json b/assets/package-lock.json index 300a9f34..69d8c483 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -2112,9 +2112,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", From 664f7c73f845b3e4aa8dd3b5d4b6262d82399836 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 17 Nov 2024 14:36:25 -0500 Subject: [PATCH 24/33] Markdown subscript migration --- assets/js/markdowntoolbar.ts | 2 +- lib/philomena/markdown.ex | 14 ++++ lib/philomena/markdown/subscript_migrator.ex | 82 ++++++++++++++++++++ lib/philomena/native.ex | 6 ++ lib/philomena/release.ex | 4 + native/philomena/Cargo.lock | 22 ++++-- native/philomena/Cargo.toml | 2 +- native/philomena/src/lib.rs | 15 +++- native/philomena/src/markdown.rs | 37 ++++++++- 9 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 lib/philomena/markdown/subscript_migrator.ex diff --git a/assets/js/markdowntoolbar.ts b/assets/js/markdowntoolbar.ts index f9ceb840..0b355e85 100644 --- a/assets/js/markdowntoolbar.ts +++ b/assets/js/markdowntoolbar.ts @@ -60,7 +60,7 @@ const markdownSyntax: Record = { }, subscript: { action: wrapSelection, - options: { prefix: '%' }, + options: { prefix: '~' }, }, quote: { action: wrapLines, diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index be3d426b..6a361c2a 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -17,6 +17,20 @@ defmodule Philomena.Markdown do def to_html_unsafe(text, replacements), do: Philomena.Native.markdown_to_html_unsafe(text, replacements) + @doc """ + Places a Markdown document into its canonical CommonMark form. + """ + @spec to_cm(String.t()) :: String.t() + def to_cm(text), + do: Philomena.Native.markdown_to_cm(text) + + @doc """ + Determines whether a Markdown document uses a subscript operator, for migration. + """ + @spec has_subscript?(String.t()) :: boolean() + def has_subscript?(text), + do: Philomena.Native.markdown_has_subscript(text) + @doc """ Escapes special characters in text which is to be rendered as Markdown. """ diff --git a/lib/philomena/markdown/subscript_migrator.ex b/lib/philomena/markdown/subscript_migrator.ex new file mode 100644 index 00000000..36d51176 --- /dev/null +++ b/lib/philomena/markdown/subscript_migrator.ex @@ -0,0 +1,82 @@ +defmodule Philomena.Markdown.SubscriptMigrator do + alias Philomena.Comments.Comment + alias Philomena.Commissions.Item, as: CommissionItem + alias Philomena.Commissions.Commission + alias Philomena.DnpEntries.DnpEntry + alias Philomena.Images.Image + alias Philomena.Conversations.Message + alias Philomena.ModNotes.ModNote + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.Tags.Tag + alias Philomena.Users.User + + import Ecto.Query + alias PhilomenaQuery.Batch + alias Philomena.Markdown + alias Philomena.Repo + + @types %{ + comments: {Comment, [:body]}, + commission_items: {CommissionItem, [:description, :add_ons]}, + commissions: {Commission, [:information, :contact, :will_create, :will_not_create]}, + dnp_entries: {DnpEntry, [:conditions, :reason, :instructions]}, + images: {Image, [:description, :scratchpad]}, + messages: {Message, [:body]}, + mod_notes: {ModNote, [:body]}, + posts: {Post, [:body]}, + reports: {Report, [:reason]}, + tags: {Tag, [:description]}, + users: {User, [:description, :scratchpad]} + } + + @doc """ + Format the ranged Markdown documents to their canonical CommonMark form. + """ + @spec migrate(type :: :all | atom(), id_start :: non_neg_integer(), id_end :: non_neg_integer()) :: + :ok + def migrate(type, id_start, id_end) + + def migrate(:all, _id_start, _id_end) do + Enum.each(@types, fn {name, _schema_columns} -> + migrate(name, 0, 2_147_483_647) + end) + end + + def migrate(type, id_start, id_end) do + IO.puts("#{type}:") + + {schema, columns} = Map.fetch!(@types, type) + + schema + |> where([s], s.id >= ^id_start and s.id < ^id_end) + |> Batch.records() + |> Enum.each(fn s -> + case generate_updates(s, columns) do + [] -> + :ok + + updates -> + IO.write("\r#{s.id}") + + {1, nil} = + schema + |> where(id: ^s.id) + |> Repo.update_all(set: updates) + end + end) + end + + @spec generate_updates(s :: struct(), columns :: [atom()]) :: Keyword.t() + defp generate_updates(s, columns) do + Enum.flat_map(columns, fn col -> + with value when not is_nil(value) <- Map.fetch!(s, col), + true <- Markdown.has_subscript?(value) do + [{col, Markdown.to_cm(value)}] + else + _ -> + [] + end + end) + end +end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index 14eeaa17..42b80c6d 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -9,6 +9,12 @@ defmodule Philomena.Native do @spec markdown_to_html_unsafe(String.t(), %{String.t() => String.t()}) :: String.t() def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded) + @spec markdown_to_cm(String.t()) :: String.t() + def markdown_to_cm(_text), do: :erlang.nif_error(:nif_not_loaded) + + @spec markdown_has_subscript(String.t()) :: boolean() + def markdown_has_subscript(_text), do: :erlang.nif_error(:nif_not_loaded) + @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena/release.ex b/lib/philomena/release.ex index 32e24263..429027cd 100644 --- a/lib/philomena/release.ex +++ b/lib/philomena/release.ex @@ -14,6 +14,10 @@ defmodule Philomena.Release do {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) end + def migrate_markdown(type, id_start, id_end) do + Philomena.Markdown.SubscriptMigrator.migrate(type, id_start, id_end) + end + def update_channels do start_app() Philomena.Channels.update_tracked_channels!() diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index bc4af0b4..295dc15c 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -34,9 +34,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bon" -version = "2.3.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869" +checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509" dependencies = [ "bon-macros", "rustversion", @@ -44,14 +44,16 @@ dependencies = [ [[package]] name = "bon-macros" -version = "2.3.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663" +checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33" dependencies = [ "darling", "ident_case", + "prettyplease", "proc-macro2", "quote", + "rustversion", "syn", ] @@ -92,7 +94,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" version = "0.29.0" -source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.1#85054b19a0383ad9c05aba1add49111c860932dc" dependencies = [ "bon", "caseless", @@ -370,6 +372,16 @@ dependencies = [ "zip", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.89" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 19d68342..de8bab3d 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["dylib"] [dependencies] base64 = "0.21" -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.1", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index ccca12a0..9f2beb5b 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -15,8 +15,9 @@ static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", [ - markdown_to_html, markdown_to_html_unsafe, camo_image_url, - zip_open_writer, zip_start_file, zip_write, zip_finish + markdown_to_html, markdown_to_html_unsafe, markdown_to_cm, + markdown_has_subscript, camo_image_url, zip_open_writer, + zip_start_file, zip_write, zip_finish ], load = load } @@ -39,6 +40,16 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String markdown::to_html_unsafe(input, reps) } +#[rustler::nif(schedule = "DirtyCpu")] +fn markdown_to_cm(input: &str) -> String { + markdown::to_cm(input) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn markdown_has_subscript(input: &str) -> bool { + markdown::has_subscript(input) +} + // Camo NIF wrappers. #[rustler::nif] diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index 778deb95..29aa89b2 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,6 +1,7 @@ use crate::{camo, domains}; -use comrak::Options; -use std::collections::HashMap; +use comrak::nodes::AstNode; +use comrak::{Arena, Options}; +use std::collections::{HashMap, VecDeque}; use std::sync::Arc; pub fn common_options() -> Options { @@ -23,6 +24,7 @@ pub fn common_options() -> Options { options.extension.greentext = true; options.extension.subscript = true; options.extension.philomena = true; + options.extension.alternate_subscript = true; options.render.ignore_empty_links = true; options.render.ignore_setext = true; @@ -52,3 +54,34 @@ pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { comrak::markdown_to_html(input, &options) } + +fn migration_options() -> Options { + let mut options = common_options(); + options.extension.subscript = false; + options +} + +pub fn to_cm(input: &str) -> String { + comrak::markdown_to_commonmark(input, &migration_options()) +} + +pub fn has_subscript(input: &str) -> bool { + let mut queue: VecDeque<&AstNode> = VecDeque::new(); + let arena = Arena::new(); + + queue.push_back(comrak::parse_document(&arena, input, &migration_options())); + + while let Some(front) = queue.pop_front() { + match &front.data.borrow().value { + comrak::nodes::NodeValue::Subscript => return true, + comrak::nodes::NodeValue::Strikethrough => return true, + _ => {} + } + + for child in front.children() { + queue.push_back(child); + } + } + + false +} From f9339292c4971b6fca0ae1a18fb48a9d39d20464 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 21 Nov 2024 09:41:39 -0500 Subject: [PATCH 25/33] Remove old subscript support --- lib/philomena/markdown.ex | 14 ---- lib/philomena/markdown/subscript_migrator.ex | 82 -------------------- lib/philomena/native.ex | 6 -- lib/philomena/release.ex | 4 - native/philomena/src/lib.rs | 15 +--- native/philomena/src/markdown.rs | 37 +-------- native/philomena/src/tests.rs | 2 +- 7 files changed, 5 insertions(+), 155 deletions(-) delete mode 100644 lib/philomena/markdown/subscript_migrator.ex diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index 6a361c2a..be3d426b 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -17,20 +17,6 @@ defmodule Philomena.Markdown do def to_html_unsafe(text, replacements), do: Philomena.Native.markdown_to_html_unsafe(text, replacements) - @doc """ - Places a Markdown document into its canonical CommonMark form. - """ - @spec to_cm(String.t()) :: String.t() - def to_cm(text), - do: Philomena.Native.markdown_to_cm(text) - - @doc """ - Determines whether a Markdown document uses a subscript operator, for migration. - """ - @spec has_subscript?(String.t()) :: boolean() - def has_subscript?(text), - do: Philomena.Native.markdown_has_subscript(text) - @doc """ Escapes special characters in text which is to be rendered as Markdown. """ diff --git a/lib/philomena/markdown/subscript_migrator.ex b/lib/philomena/markdown/subscript_migrator.ex deleted file mode 100644 index 36d51176..00000000 --- a/lib/philomena/markdown/subscript_migrator.ex +++ /dev/null @@ -1,82 +0,0 @@ -defmodule Philomena.Markdown.SubscriptMigrator do - alias Philomena.Comments.Comment - alias Philomena.Commissions.Item, as: CommissionItem - alias Philomena.Commissions.Commission - alias Philomena.DnpEntries.DnpEntry - alias Philomena.Images.Image - alias Philomena.Conversations.Message - alias Philomena.ModNotes.ModNote - alias Philomena.Posts.Post - alias Philomena.Reports.Report - alias Philomena.Tags.Tag - alias Philomena.Users.User - - import Ecto.Query - alias PhilomenaQuery.Batch - alias Philomena.Markdown - alias Philomena.Repo - - @types %{ - comments: {Comment, [:body]}, - commission_items: {CommissionItem, [:description, :add_ons]}, - commissions: {Commission, [:information, :contact, :will_create, :will_not_create]}, - dnp_entries: {DnpEntry, [:conditions, :reason, :instructions]}, - images: {Image, [:description, :scratchpad]}, - messages: {Message, [:body]}, - mod_notes: {ModNote, [:body]}, - posts: {Post, [:body]}, - reports: {Report, [:reason]}, - tags: {Tag, [:description]}, - users: {User, [:description, :scratchpad]} - } - - @doc """ - Format the ranged Markdown documents to their canonical CommonMark form. - """ - @spec migrate(type :: :all | atom(), id_start :: non_neg_integer(), id_end :: non_neg_integer()) :: - :ok - def migrate(type, id_start, id_end) - - def migrate(:all, _id_start, _id_end) do - Enum.each(@types, fn {name, _schema_columns} -> - migrate(name, 0, 2_147_483_647) - end) - end - - def migrate(type, id_start, id_end) do - IO.puts("#{type}:") - - {schema, columns} = Map.fetch!(@types, type) - - schema - |> where([s], s.id >= ^id_start and s.id < ^id_end) - |> Batch.records() - |> Enum.each(fn s -> - case generate_updates(s, columns) do - [] -> - :ok - - updates -> - IO.write("\r#{s.id}") - - {1, nil} = - schema - |> where(id: ^s.id) - |> Repo.update_all(set: updates) - end - end) - end - - @spec generate_updates(s :: struct(), columns :: [atom()]) :: Keyword.t() - defp generate_updates(s, columns) do - Enum.flat_map(columns, fn col -> - with value when not is_nil(value) <- Map.fetch!(s, col), - true <- Markdown.has_subscript?(value) do - [{col, Markdown.to_cm(value)}] - else - _ -> - [] - end - end) - end -end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index 42b80c6d..14eeaa17 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -9,12 +9,6 @@ defmodule Philomena.Native do @spec markdown_to_html_unsafe(String.t(), %{String.t() => String.t()}) :: String.t() def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded) - @spec markdown_to_cm(String.t()) :: String.t() - def markdown_to_cm(_text), do: :erlang.nif_error(:nif_not_loaded) - - @spec markdown_has_subscript(String.t()) :: boolean() - def markdown_has_subscript(_text), do: :erlang.nif_error(:nif_not_loaded) - @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena/release.ex b/lib/philomena/release.ex index 429027cd..32e24263 100644 --- a/lib/philomena/release.ex +++ b/lib/philomena/release.ex @@ -14,10 +14,6 @@ defmodule Philomena.Release do {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) end - def migrate_markdown(type, id_start, id_end) do - Philomena.Markdown.SubscriptMigrator.migrate(type, id_start, id_end) - end - def update_channels do start_app() Philomena.Channels.update_tracked_channels!() diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 9f2beb5b..ccca12a0 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -15,9 +15,8 @@ static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", [ - markdown_to_html, markdown_to_html_unsafe, markdown_to_cm, - markdown_has_subscript, camo_image_url, zip_open_writer, - zip_start_file, zip_write, zip_finish + markdown_to_html, markdown_to_html_unsafe, camo_image_url, + zip_open_writer, zip_start_file, zip_write, zip_finish ], load = load } @@ -40,16 +39,6 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String markdown::to_html_unsafe(input, reps) } -#[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_cm(input: &str) -> String { - markdown::to_cm(input) -} - -#[rustler::nif(schedule = "DirtyCpu")] -fn markdown_has_subscript(input: &str) -> bool { - markdown::has_subscript(input) -} - // Camo NIF wrappers. #[rustler::nif] diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index 29aa89b2..778deb95 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,7 +1,6 @@ use crate::{camo, domains}; -use comrak::nodes::AstNode; -use comrak::{Arena, Options}; -use std::collections::{HashMap, VecDeque}; +use comrak::Options; +use std::collections::HashMap; use std::sync::Arc; pub fn common_options() -> Options { @@ -24,7 +23,6 @@ pub fn common_options() -> Options { options.extension.greentext = true; options.extension.subscript = true; options.extension.philomena = true; - options.extension.alternate_subscript = true; options.render.ignore_empty_links = true; options.render.ignore_setext = true; @@ -54,34 +52,3 @@ pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { comrak::markdown_to_html(input, &options) } - -fn migration_options() -> Options { - let mut options = common_options(); - options.extension.subscript = false; - options -} - -pub fn to_cm(input: &str) -> String { - comrak::markdown_to_commonmark(input, &migration_options()) -} - -pub fn has_subscript(input: &str) -> bool { - let mut queue: VecDeque<&AstNode> = VecDeque::new(); - let arena = Arena::new(); - - queue.push_back(comrak::parse_document(&arena, input, &migration_options())); - - while let Some(front) = queue.pop_front() { - match &front.data.borrow().value { - comrak::nodes::NodeValue::Subscript => return true, - comrak::nodes::NodeValue::Strikethrough => return true, - _ => {} - } - - for child in front.children() { - queue.push_back(child); - } - } - - false -} diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs index f57eb0f2..39374cee 100644 --- a/native/philomena/src/tests.rs +++ b/native/philomena/src/tests.rs @@ -45,7 +45,7 @@ fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) { #[test] fn subscript() { - html("H%2%O\n", "
H2O
\n"); + html("H~2~O\n", "
H2O
\n"); } #[test] From e8a67d0a1e69af31597de76bc29d0dc7e510a6c0 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 21 Nov 2024 09:46:32 -0500 Subject: [PATCH 26/33] Update comrak version for autolinks --- native/philomena/Cargo.lock | 2 +- native/philomena/Cargo.toml | 2 +- native/philomena/src/tests.rs | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 295dc15c..4ef3513f 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -94,7 +94,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" version = "0.29.0" -source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.1#85054b19a0383ad9c05aba1add49111c860932dc" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.2#00ac2a12d5797feb0ceba9a98487451ab65593fe" dependencies = [ "bon", "caseless", diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index de8bab3d..7d8366f1 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["dylib"] [dependencies] base64 = "0.21" -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.1", default-features = false } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs index 39374cee..84418ed4 100644 --- a/native/philomena/src/tests.rs +++ b/native/philomena/src/tests.rs @@ -56,6 +56,14 @@ fn subscript_autolink_interaction() { ); } +#[test] +fn underscore_autolink_interaction() { + html( + "https://example.com/x_", + "
https://example.com/x_
\n" + ) +} + #[test] fn spoiler() { html( From 241744bec6109da4b5e6ef73ace441c5b1c89f87 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 21 Nov 2024 13:08:13 -0500 Subject: [PATCH 27/33] Fix help toolbar --- lib/philomena_web/templates/markdown/_help.html.slime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/philomena_web/templates/markdown/_help.html.slime b/lib/philomena_web/templates/markdown/_help.html.slime index e2838ea3..08648c5a 100644 --- a/lib/philomena_web/templates/markdown/_help.html.slime +++ b/lib/philomena_web/templates/markdown/_help.html.slime @@ -9,7 +9,7 @@ ins> __underline__ del> ~~strike~~ sup> ^sup^ - sub %sub% + sub ~sub~ p a href="/pages/markdown" From 50e0f46e06653f1576c3d053f843acf515e382fb Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 15:14:58 -0500 Subject: [PATCH 28/33] Fix clippy for 1.83 --- native/philomena/src/zip.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs index 60ef15be..47fd4977 100644 --- a/native/philomena/src/zip.rs +++ b/native/philomena/src/zip.rs @@ -18,6 +18,8 @@ pub struct WriterResource { pub type WriterResourceArc = ResourceArc; +// TODO: rustler must fix the resource macro to provide a top-level impl definiton +#[allow(non_local_definitions)] pub fn load(env: Env, _: Term) -> bool { rustler::resource!(WriterResource, env); true From b45130f073bbf71674f2c0669475dd3813e9300d Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 15:16:02 -0500 Subject: [PATCH 29/33] Typo --- native/philomena/src/zip.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs index 47fd4977..9d8c35aa 100644 --- a/native/philomena/src/zip.rs +++ b/native/philomena/src/zip.rs @@ -18,7 +18,7 @@ pub struct WriterResource { pub type WriterResourceArc = ResourceArc; -// TODO: rustler must fix the resource macro to provide a top-level impl definiton +// TODO: rustler must fix the resource macro to provide a top-level impl definition #[allow(non_local_definitions)] pub fn load(env: Env, _: Term) -> bool { rustler::resource!(WriterResource, env); From 9649d0da5245a54be8680b34afac78a651ca2f1a Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 16:49:16 -0500 Subject: [PATCH 30/33] Fix rustler warning --- mix.exs | 2 +- mix.lock | 12 ++-- native/philomena/Cargo.lock | 135 +++++++++++++++++++++++++----------- native/philomena/Cargo.toml | 2 +- native/philomena/src/lib.rs | 15 +--- native/philomena/src/zip.rs | 12 ++-- 6 files changed, 110 insertions(+), 68 deletions(-) diff --git a/mix.exs b/mix.exs index 4032d859..1dde7ea0 100644 --- a/mix.exs +++ b/mix.exs @@ -77,7 +77,7 @@ defmodule Philomena.MixProject do {:mail, "~> 0.3.0"}, # Markdown - {:rustler, "~> 0.27"}, + {:rustler, "~> 0.35"}, # Linting {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 7f179ae0..35d487fe 100644 --- a/mix.lock +++ b/mix.lock @@ -5,14 +5,14 @@ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "canada": {:hex, :canada, "1.0.2", "040e4c47609b0a67d5773ac1fbe5e99f840cef173d69b739beda7c98453e0770", [:mix], [], "hexpm", "4269f74153fe89583fe50bd4d5de57bfe01f31258a6b676d296f3681f1483c68"}, "canary": {:git, "https://github.com/marcinkoziej/canary.git", "704debde7a2c0600f78c687807884bf37c45bd79", [ref: "704debde7a2c0600f78c687807884bf37c45bd79"]}, - "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, "credo_envvar": {:hex, :credo_envvar, "0.1.4", "40817c10334e400f031012c0510bfa0d8725c19d867e4ae39cf14f2cbebc3b20", [:mix], [{:credo, "~> 1.0", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "5055cdb4bcbaf7d423bc2bb3ac62b4e2d825e2b1e816884c468dee59d0363009"}, "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, "ecto": {:hex, :ecto, "3.12.1", "626765f7066589de6fa09e0876a253ff60c3d00870dd3a1cd696e2ba67bfceea", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df0045ab9d87be947228e05a8d153f3e06e0d05ab10c3b3cc557d2f7243d1940"}, @@ -27,7 +27,7 @@ "expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"}, "exq": {:hex, :exq, "0.19.0", "06eb92944dad39f0954dc8f63190d3e24d11734eef88cf5800883e57ebf74f3c", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 6.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "24fc0ebdd87cc7406e1034fb46c2419f9c8a362f0ec634d23b6b819514d36390"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, "gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, @@ -62,8 +62,8 @@ "qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"}, "redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, - "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"}, - "rustler": {:hex, :rustler, "0.34.0", "e9a73ee419fc296a10e49b415a2eb87a88c9217aa0275ec9f383d37eed290c1c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "1d0c7449482b459513003230c0e2422b0252245776fe6fd6e41cb2b11bd8e628"}, + "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"}, + "rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]}, "secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"}, @@ -71,7 +71,7 @@ "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"}, "swoosh": {:hex, :swoosh, "1.16.10", "04be6e2eb1a31aa0aa21a731175c81cc3998189456a92daf13d44a5c754afcf5", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "756be04db173c0cbe318f1dfe2bcc88aa63aed78cf5a4b02b61b36ee11fc716a"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 4ef3513f..d2481f12 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -230,9 +230,9 @@ checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "http" @@ -271,6 +271,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + [[package]] name = "itoa" version = "1.0.11" @@ -306,18 +312,22 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + [[package]] name = "lockfree-object-pool" version = "0.1.6" @@ -423,6 +433,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.3" @@ -446,37 +462,29 @@ dependencies = [ [[package]] name = "rustler" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7a2f98cb272ae0548e434bb3afde626012084cbebef84542bc7afed868bd6f" +checksum = "b705f2c3643cc170d8888cb6bad589155d9c0248f3104ef7a04c2b7ffbaf13fc" dependencies = [ - "lazy_static", + "inventory", + "libloading", + "regex-lite", "rustler_codegen", - "rustler_sys", ] [[package]] name = "rustler_codegen" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae37fb154683a1ff13e95dfd16c15043d6aee9853ea650ca34a45bd94e319a5" +checksum = "3ad56caff00562948bd6ac33c18dbc579e5a1bbee2d7f2f54073307e57f6b57a" dependencies = [ "heck", + "inventory", "proc-macro2", "quote", "syn", ] -[[package]] -name = "rustler_sys" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0eb19e2fdf4abc3662441087fc499a58b7550822cc8007e79f15c40157c883" -dependencies = [ - "regex", - "unreachable", -] - [[package]] name = "rustversion" version = "1.0.18" @@ -590,15 +598,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -616,12 +615,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -708,6 +701,70 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "zip" version = "2.2.0" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 7d8366f1..4ff4467d 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -16,7 +16,7 @@ http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" ring = "0.16" -rustler = "0.28" +rustler = "0.35" url = "2.3" zip = { version = "2.2.0", features = ["deflate"], default-features = false } diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index ccca12a0..6f7f72b2 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,5 +1,5 @@ use jemallocator::Jemalloc; -use rustler::{Atom, Binary, Env, Term}; +use rustler::{Atom, Binary}; use std::collections::HashMap; mod camo; @@ -13,18 +13,7 @@ mod zip; static GLOBAL: Jemalloc = Jemalloc; rustler::init! { - "Elixir.Philomena.Native", - [ - markdown_to_html, markdown_to_html_unsafe, camo_image_url, - zip_open_writer, zip_start_file, zip_write, zip_finish - ], - load = load -} - -// Setup. - -fn load(env: Env, arg: Term) -> bool { - zip::load(env, arg) + "Elixir.Philomena.Native" } // Markdown NIF wrappers. diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs index 9d8c35aa..069246a3 100644 --- a/native/philomena/src/zip.rs +++ b/native/philomena/src/zip.rs @@ -2,7 +2,7 @@ use std::fs::{File, OpenOptions}; use std::io::Write; use std::sync::Mutex; -use rustler::{Atom, Env, ResourceArc, Term}; +use rustler::{Atom, Resource, ResourceArc}; use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; mod atoms { @@ -16,14 +16,10 @@ pub struct WriterResource { inner: Mutex>>, } -pub type WriterResourceArc = ResourceArc; +#[rustler::resource_impl] +impl Resource for WriterResource {} -// TODO: rustler must fix the resource macro to provide a top-level impl definition -#[allow(non_local_definitions)] -pub fn load(env: Env, _: Term) -> bool { - rustler::resource!(WriterResource, env); - true -} +pub type WriterResourceArc = ResourceArc; fn with_writer(writer: WriterResourceArc, f: F) -> Atom where From 87e0c814af4d87d5b049659001ac131dc022b452 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 17:30:18 -0500 Subject: [PATCH 31/33] remove unnecessary move --- native/philomena/src/zip.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs index 069246a3..24afd300 100644 --- a/native/philomena/src/zip.rs +++ b/native/philomena/src/zip.rs @@ -53,15 +53,15 @@ pub fn open_writer(path: &str) -> Result { pub fn start_file(writer: WriterResourceArc, name: &str) -> Atom { let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); - with_writer(writer, move |writer| { + with_writer(writer, |writer| { writer.as_mut()?.start_file(name, options).ok() }) } pub fn write(writer: WriterResourceArc, data: &[u8]) -> Atom { - with_writer(writer, move |writer| writer.as_mut()?.write(data).ok()) + with_writer(writer, |writer| writer.as_mut()?.write(data).ok()) } pub fn finish(writer: WriterResourceArc) -> Atom { - with_writer(writer, move |writer| writer.take().map(|w| w.finish().ok())) + with_writer(writer, |writer| writer.take().map(|w| w.finish().ok())) } From 4d9edf636418b861e9e80c9a64fdeb2ce502ea11 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 18:03:34 -0500 Subject: [PATCH 32/33] Remove unnecessary deref coercion --- native/philomena/src/zip.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/philomena/src/zip.rs b/native/philomena/src/zip.rs index 24afd300..14489ee2 100644 --- a/native/philomena/src/zip.rs +++ b/native/philomena/src/zip.rs @@ -25,7 +25,7 @@ fn with_writer(writer: WriterResourceArc, f: F) -> Atom where F: FnOnce(&mut Option>) -> Option, { - let mut guard = match (*writer).inner.lock() { + let mut guard = match writer.inner.lock() { Ok(g) => g, Err(_) => return atoms::error(), }; From 0a0bcd15bf369de68b27a151e43793e95394552f Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 1 Dec 2024 18:17:53 -0500 Subject: [PATCH 33/33] Use lazy for domain set --- native/philomena/Cargo.lock | 5 +++-- native/philomena/Cargo.toml | 1 + native/philomena/src/domains.rs | 28 ++++++++++++++++------------ native/philomena/src/markdown.rs | 4 ++-- native/philomena/src/tests.rs | 4 ++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index d2481f12..54aa5826 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -357,9 +357,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "percent-encoding" @@ -375,6 +375,7 @@ dependencies = [ "comrak", "http", "jemallocator", + "once_cell", "regex", "ring", "rustler", diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 4ff4467d..e6bba775 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -14,6 +14,7 @@ base64 = "0.21" comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } +once_cell = "1.20" regex = "1" ring = "0.16" rustler = "0.35" diff --git a/native/philomena/src/domains.rs b/native/philomena/src/domains.rs index c5626c12..9bcacfc9 100644 --- a/native/philomena/src/domains.rs +++ b/native/philomena/src/domains.rs @@ -1,25 +1,29 @@ -use http::Uri; -use regex::Regex; +use std::collections::BTreeSet; use std::env; -pub fn get() -> Option> { +use http::Uri; +use once_cell::sync::Lazy; +use regex::Regex; + +pub type DomainSet = BTreeSet; + +static DOMAINS: Lazy> = Lazy::new(|| { if let Ok(domains) = env::var("SITE_DOMAINS") { - return Some( - domains - .split(',') - .map(|s| s.to_string()) - .collect::>(), - ); + return Some(domains.split(',').map(|s| s.to_string()).collect()); } None +}); + +pub fn get() -> &'static Option { + &DOMAINS } -pub fn relativize(domains: &[String], url: &str) -> Option { +pub fn relativize(domains: &DomainSet, url: &str) -> Option { let uri = url.parse::().ok()?; if let Some(a) = uri.authority() { - if domains.contains(&a.host().to_string()) { + if domains.contains(a.host()) { if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) { return Some(re.replace(url, "").into()); } @@ -29,6 +33,6 @@ pub fn relativize(domains: &[String], url: &str) -> Option { Some(url.into()) } -pub fn relativize_careful(domains: &[String], url: &str) -> String { +pub fn relativize_careful(domains: &DomainSet, url: &str) -> String { relativize(domains, url).unwrap_or_else(|| url.into()) } diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index 778deb95..fbd846ac 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -29,8 +29,8 @@ pub fn common_options() -> Options { options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url))); if let Some(domains) = domains::get() { - options.extension.link_url_rewriter = Some(Arc::new(move |url: &str| { - domains::relativize_careful(&domains, url) + options.extension.link_url_rewriter = Some(Arc::new(|url: &str| { + domains::relativize_careful(domains, url) })); } diff --git a/native/philomena/src/tests.rs b/native/philomena/src/tests.rs index 84418ed4..547f0f86 100644 --- a/native/philomena/src/tests.rs +++ b/native/philomena/src/tests.rs @@ -236,8 +236,8 @@ fn image_mention_line_start() { #[test] fn auto_relative_links() { - let domains: Vec = vec!["example.com".into()]; - let f = Arc::new(move |url: &str| domains::relativize_careful(&domains, url)); + let domains = Arc::new(vec!["example.com".into()].into_iter().collect()); + let f = Arc::new(move |url: &str| domains::relativize_careful(&*domains, url)); html_opts_i( "[some link text](https://example.com/some/path)",