Merge pull request #374 from philomena-dev/subscript-migration

Markdown subscript migration

assets/js/markdowntoolbar.ts

@@ -60,7 +60,7 @@ const markdownSyntax: Record<string, SyntaxHandler> = {
   },
   subscript: {
     action: wrapSelection,
-    options: { prefix: '%' },
+    options: { prefix: '~' },
   },
   quote: {
     action: wrapLines,

lib/philomena/markdown.ex

@@ -17,6 +17,20 @@ defmodule Philomena.Markdown do
   def to_html_unsafe(text, replacements),
     do: Philomena.Native.markdown_to_html_unsafe(text, replacements)
 
+  @doc """
+  Places a Markdown document into its canonical CommonMark form.
+  """
+  @spec to_cm(String.t()) :: String.t()
+  def to_cm(text),
+    do: Philomena.Native.markdown_to_cm(text)
+
+  @doc """
+  Determines whether a Markdown document uses a subscript operator, for migration.
+  """
+  @spec has_subscript?(String.t()) :: boolean()
+  def has_subscript?(text),
+    do: Philomena.Native.markdown_has_subscript(text)
+
   @doc """
   Escapes special characters in text which is to be rendered as Markdown.
   """

lib/philomena/markdown/subscript_migrator.ex

@@ -0,0 +1,82 @@
+defmodule Philomena.Markdown.SubscriptMigrator do
+  alias Philomena.Comments.Comment
+  alias Philomena.Commissions.Item, as: CommissionItem
+  alias Philomena.Commissions.Commission
+  alias Philomena.DnpEntries.DnpEntry
+  alias Philomena.Images.Image
+  alias Philomena.Conversations.Message
+  alias Philomena.ModNotes.ModNote
+  alias Philomena.Posts.Post
+  alias Philomena.Reports.Report
+  alias Philomena.Tags.Tag
+  alias Philomena.Users.User
+
+  import Ecto.Query
+  alias PhilomenaQuery.Batch
+  alias Philomena.Markdown
+  alias Philomena.Repo
+
+  @types %{
+    comments: {Comment, [:body]},
+    commission_items: {CommissionItem, [:description, :add_ons]},
+    commissions: {Commission, [:information, :contact, :will_create, :will_not_create]},
+    dnp_entries: {DnpEntry, [:conditions, :reason, :instructions]},
+    images: {Image, [:description, :scratchpad]},
+    messages: {Message, [:body]},
+    mod_notes: {ModNote, [:body]},
+    posts: {Post, [:body]},
+    reports: {Report, [:reason]},
+    tags: {Tag, [:description]},
+    users: {User, [:description, :scratchpad]}
+  }
+
+  @doc """
+  Format the ranged Markdown documents to their canonical CommonMark form.
+  """
+  @spec migrate(type :: :all | atom(), id_start :: non_neg_integer(), id_end :: non_neg_integer()) ::
+          :ok
+  def migrate(type, id_start, id_end)
+
+  def migrate(:all, _id_start, _id_end) do
+    Enum.each(@types, fn {name, _schema_columns} ->
+      migrate(name, 0, 2_147_483_647)
+    end)
+  end
+
+  def migrate(type, id_start, id_end) do
+    IO.puts("#{type}:")
+
+    {schema, columns} = Map.fetch!(@types, type)
+
+    schema
+    |> where([s], s.id >= ^id_start and s.id < ^id_end)
+    |> Batch.records()
+    |> Enum.each(fn s ->
+      case generate_updates(s, columns) do
+        [] ->
+          :ok
+
+        updates ->
+          IO.write("\r#{s.id}")
+
+          {1, nil} =
+            schema
+            |> where(id: ^s.id)
+            |> Repo.update_all(set: updates)
+      end
+    end)
+  end
+
+  @spec generate_updates(s :: struct(), columns :: [atom()]) :: Keyword.t()
+  defp generate_updates(s, columns) do
+    Enum.flat_map(columns, fn col ->
+      with value when not is_nil(value) <- Map.fetch!(s, col),
+           true <- Markdown.has_subscript?(value) do
+        [{col, Markdown.to_cm(value)}]
+      else
+        _ ->
+          []
+      end
+    end)
+  end
+end

lib/philomena/native.ex

@@ -9,6 +9,12 @@ defmodule Philomena.Native do
   @spec markdown_to_html_unsafe(String.t(), %{String.t() => String.t()}) :: String.t()
   def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded)
 
+  @spec markdown_to_cm(String.t()) :: String.t()
+  def markdown_to_cm(_text), do: :erlang.nif_error(:nif_not_loaded)
+
+  @spec markdown_has_subscript(String.t()) :: boolean()
+  def markdown_has_subscript(_text), do: :erlang.nif_error(:nif_not_loaded)
+
   @spec camo_image_url(String.t()) :: String.t()
   def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
 

lib/philomena/release.ex

@@ -14,6 +14,10 @@ defmodule Philomena.Release do
     {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
   end
 
+  def migrate_markdown(type, id_start, id_end) do
+    Philomena.Markdown.SubscriptMigrator.migrate(type, id_start, id_end)
+  end
+
   def update_channels do
     start_app()
     Philomena.Channels.update_tracked_channels!()

native/philomena/Cargo.lock

@@ -34,9 +34,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "bon"
-version = "2.3.0"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869"
+checksum = "a636f83af97c6946f3f5cf5c268ec02375bf5efd371110292dfd57961f57a509"
 dependencies = [
  "bon-macros",
  "rustversion",
@@ -44,14 +44,16 @@ dependencies = [
 
 [[package]]
 name = "bon-macros"
-version = "2.3.0"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663"
+checksum = "a7eaf1bfaa5b8d512abfd36d0c432591fef139d3de9ee54f1f839ea109d70d33"
 dependencies = [
  "darling",
  "ident_case",
+ "prettyplease",
  "proc-macro2",
  "quote",
+ "rustversion",
  "syn",
 ]
 
@@ -92,7 +94,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 [[package]]
 name = "comrak"
 version = "0.29.0"
-source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e"
+source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.1#85054b19a0383ad9c05aba1add49111c860932dc"
 dependencies = [
  "bon",
  "caseless",
@@ -370,6 +372,16 @@ dependencies = [
  "zip",
 ]
 
+[[package]]
+name = "prettyplease"
+version = "0.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.89"

native/philomena/Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["dylib"]
 
 [dependencies]
 base64 = "0.21"
-comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false }
+comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.1", default-features = false }
 http = "0.2"
 jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
 regex = "1"

native/philomena/src/lib.rs

@@ -15,8 +15,9 @@ static GLOBAL: Jemalloc = Jemalloc;
 rustler::init! {
     "Elixir.Philomena.Native",
     [
-        markdown_to_html, markdown_to_html_unsafe, camo_image_url,
-        zip_open_writer, zip_start_file, zip_write, zip_finish
+        markdown_to_html, markdown_to_html_unsafe, markdown_to_cm,
+        markdown_has_subscript, camo_image_url, zip_open_writer,
+        zip_start_file, zip_write, zip_finish
     ],
     load = load
 }
@@ -39,6 +40,16 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String
     markdown::to_html_unsafe(input, reps)
 }
 
+#[rustler::nif(schedule = "DirtyCpu")]
+fn markdown_to_cm(input: &str) -> String {
+    markdown::to_cm(input)
+}
+
+#[rustler::nif(schedule = "DirtyCpu")]
+fn markdown_has_subscript(input: &str) -> bool {
+    markdown::has_subscript(input)
+}
+
 // Camo NIF wrappers.
 
 #[rustler::nif]

native/philomena/src/markdown.rs

@@ -1,6 +1,7 @@
 use crate::{camo, domains};
-use comrak::Options;
-use std::collections::HashMap;
+use comrak::nodes::AstNode;
+use comrak::{Arena, Options};
+use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 
 pub fn common_options() -> Options {
@@ -23,6 +24,7 @@ pub fn common_options() -> Options {
     options.extension.greentext = true;
     options.extension.subscript = true;
     options.extension.philomena = true;
+    options.extension.alternate_subscript = true;
     options.render.ignore_empty_links = true;
     options.render.ignore_setext = true;
 
@@ -52,3 +54,34 @@ pub fn to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String {
 
     comrak::markdown_to_html(input, &options)
 }
+
+fn migration_options() -> Options {
+    let mut options = common_options();
+    options.extension.subscript = false;
+    options
+}
+
+pub fn to_cm(input: &str) -> String {
+    comrak::markdown_to_commonmark(input, &migration_options())
+}
+
+pub fn has_subscript(input: &str) -> bool {
+    let mut queue: VecDeque<&AstNode> = VecDeque::new();
+    let arena = Arena::new();
+
+    queue.push_back(comrak::parse_document(&arena, input, &migration_options()));
+
+    while let Some(front) = queue.pop_front() {
+        match &front.data.borrow().value {
+            comrak::nodes::NodeValue::Subscript => return true,
+            comrak::nodes::NodeValue::Strikethrough => return true,
+            _ => {}
+        }
+
+        for child in front.children() {
+            queue.push_back(child);
+        }
+    }
+
+    false
+}