diff --git a/assets/js/markdowntoolbar.ts b/assets/js/markdowntoolbar.ts index f9ceb840..0b355e85 100644 --- a/assets/js/markdowntoolbar.ts +++ b/assets/js/markdowntoolbar.ts @@ -60,7 +60,7 @@ const markdownSyntax: Record = { }, subscript: { action: wrapSelection, - options: { prefix: '%' }, + options: { prefix: '~' }, }, quote: { action: wrapLines, diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index be3d426b..6a361c2a 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -17,6 +17,20 @@ defmodule Philomena.Markdown do def to_html_unsafe(text, replacements), do: Philomena.Native.markdown_to_html_unsafe(text, replacements) + @doc """ + Places a Markdown document into its canonical CommonMark form. + """ + @spec to_cm(String.t()) :: String.t() + def to_cm(text), + do: Philomena.Native.markdown_to_cm(text) + + @doc """ + Determines whether a Markdown document uses a subscript operator, for migration. + """ + @spec has_subscript?(String.t()) :: boolean() + def has_subscript?(text), + do: Philomena.Native.markdown_has_subscript(text) + @doc """ Escapes special characters in text which is to be rendered as Markdown. """ diff --git a/lib/philomena/markdown/subscript_migrator.ex b/lib/philomena/markdown/subscript_migrator.ex new file mode 100644 index 00000000..36d51176 --- /dev/null +++ b/lib/philomena/markdown/subscript_migrator.ex @@ -0,0 +1,82 @@ +defmodule Philomena.Markdown.SubscriptMigrator do + alias Philomena.Comments.Comment + alias Philomena.Commissions.Item, as: CommissionItem + alias Philomena.Commissions.Commission + alias Philomena.DnpEntries.DnpEntry + alias Philomena.Images.Image + alias Philomena.Conversations.Message + alias Philomena.ModNotes.ModNote + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.Tags.Tag + alias Philomena.Users.User + + import Ecto.Query + alias PhilomenaQuery.Batch + alias Philomena.Markdown + alias Philomena.Repo + + @types %{ + comments: {Comment, [:body]}, + commission_items: {CommissionItem, [:description, :add_ons]}, + commissions: {Commission, [:information, :contact, :will_create, :will_not_create]}, + dnp_entries: {DnpEntry, [:conditions, :reason, :instructions]}, + images: {Image, [:description, :scratchpad]}, + messages: {Message, [:body]}, + mod_notes: {ModNote, [:body]}, + posts: {Post, [:body]}, + reports: {Report, [:reason]}, + tags: {Tag, [:description]}, + users: {User, [:description, :scratchpad]} + } + + @doc """ + Format the ranged Markdown documents to their canonical CommonMark form. + """ + @spec migrate(type :: :all | atom(), id_start :: non_neg_integer(), id_end :: non_neg_integer()) :: + :ok + def migrate(type, id_start, id_end) + + def migrate(:all, _id_start, _id_end) do + Enum.each(@types, fn {name, _schema_columns} -> + migrate(name, 0, 2_147_483_647) + end) + end + + def migrate(type, id_start, id_end) do + IO.puts("#{type}:") + + {schema, columns} = Map.fetch!(@types, type) + + schema + |> where([s], s.id >= ^id_start and s.id < ^id_end) + |> Batch.records() + |> Enum.each(fn s -> + case generate_updates(s, columns) do + [] -> + :ok + + updates -> + IO.write("\r#{s.id}") + + {1, nil} = + schema + |> where(id: ^s.id) + |> Repo.update_all(set: updates) + end + end) + end + + @spec generate_updates(s :: struct(), columns :: [atom()]) :: Keyword.t() + defp generate_updates(s, columns) do + Enum.flat_map(columns, fn col -> + with value when not is_nil(value) <- Map.fetch!(s, col), + true <- Markdown.has_subscript?(value) do + [{col, Markdown.to_cm(value)}] + else + _ -> + [] + end + end) + end +end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index 14eeaa17..42b80c6d 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -9,6 +9,12 @@ defmodule Philomena.Native do @spec markdown_to_html_unsafe(String.t(), %{String.t() => String.t()}) :: String.t() def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded) + @spec markdown_to_cm(String.t()) :: String.t() + def markdown_to_cm(_text), do: :erlang.nif_error(:nif_not_loaded) + + @spec markdown_has_subscript(String.t()) :: boolean() + def markdown_has_subscript(_text), do: :erlang.nif_error(:nif_not_loaded) + @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena/release.ex b/lib/philomena/release.ex index 32e24263..429027cd 100644 --- a/lib/philomena/release.ex +++ b/lib/philomena/release.ex @@ -14,6 +14,10 @@ defmodule Philomena.Release do {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) end + def migrate_markdown(type, id_start, id_end) do + Philomena.Markdown.SubscriptMigrator.migrate(type, id_start, id_end) + end + def update_channels do start_app() Philomena.Channels.update_tracked_channels!() diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index bc4af0b4..295dc15c 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -34,9 +34,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bon" -version = "2.3.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869" +checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509" dependencies = [ "bon-macros", "rustversion", @@ -44,14 +44,16 @@ dependencies = [ [[package]] name = "bon-macros" -version = "2.3.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663" +checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33" dependencies = [ "darling", "ident_case", + "prettyplease", "proc-macro2", "quote", + "rustversion", "syn", ] @@ -92,7 +94,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "comrak" version = "0.29.0" -source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e" +source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.1#85054b19a0383ad9c05aba1add49111c860932dc" dependencies = [ "bon", "caseless", @@ -370,6 +372,16 @@ dependencies = [ "zip", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.89" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 19d68342..de8bab3d 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["dylib"] [dependencies] base64 = "0.21" -comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.1", default-features = false } http = "0.2" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } regex = "1" diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index ccca12a0..9f2beb5b 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -15,8 +15,9 @@ static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", [ - markdown_to_html, markdown_to_html_unsafe, camo_image_url, - zip_open_writer, zip_start_file, zip_write, zip_finish + markdown_to_html, markdown_to_html_unsafe, markdown_to_cm, + markdown_has_subscript, camo_image_url, zip_open_writer, + zip_start_file, zip_write, zip_finish ], load = load } @@ -39,6 +40,16 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap) -> String markdown::to_html_unsafe(input, reps) } +#[rustler::nif(schedule = "DirtyCpu")] +fn markdown_to_cm(input: &str) -> String { + markdown::to_cm(input) +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn markdown_has_subscript(input: &str) -> bool { + markdown::has_subscript(input) +} + // Camo NIF wrappers. #[rustler::nif] diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index 778deb95..29aa89b2 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,6 +1,7 @@ use crate::{camo, domains}; -use comrak::Options; -use std::collections::HashMap; +use comrak::nodes::AstNode; +use comrak::{Arena, Options}; +use std::collections::{HashMap, VecDeque}; use std::sync::Arc; pub fn common_options() -> Options { @@ -23,6 +24,7 @@ pub fn common_options() -> Options { options.extension.greentext = true; options.extension.subscript = true; options.extension.philomena = true; + options.extension.alternate_subscript = true; options.render.ignore_empty_links = true; options.render.ignore_setext = true; @@ -52,3 +54,34 @@ pub fn to_html_unsafe(input: &str, reps: HashMap) -> String { comrak::markdown_to_html(input, &options) } + +fn migration_options() -> Options { + let mut options = common_options(); + options.extension.subscript = false; + options +} + +pub fn to_cm(input: &str) -> String { + comrak::markdown_to_commonmark(input, &migration_options()) +} + +pub fn has_subscript(input: &str) -> bool { + let mut queue: VecDeque<&AstNode> = VecDeque::new(); + let arena = Arena::new(); + + queue.push_back(comrak::parse_document(&arena, input, &migration_options())); + + while let Some(front) = queue.pop_front() { + match &front.data.borrow().value { + comrak::nodes::NodeValue::Subscript => return true, + comrak::nodes::NodeValue::Strikethrough => return true, + _ => {} + } + + for child in front.children() { + queue.push_back(child); + } + } + + false +}