Merge pull request #375 from philomena-dev/comrak-post-migration

Comrak post migration

lib/philomena/markdown.ex

@@ -17,20 +17,6 @@ defmodule Philomena.Markdown do
   def to_html_unsafe(text, replacements),
     do: Philomena.Native.markdown_to_html_unsafe(text, replacements)
 
-  @doc """
-  Places a Markdown document into its canonical CommonMark form.
-  """
-  @spec to_cm(String.t()) :: String.t()
-  def to_cm(text),
-    do: Philomena.Native.markdown_to_cm(text)
-
-  @doc """
-  Determines whether a Markdown document uses a subscript operator, for migration.
-  """
-  @spec has_subscript?(String.t()) :: boolean()
-  def has_subscript?(text),
-    do: Philomena.Native.markdown_has_subscript(text)
-
   @doc """
   Escapes special characters in text which is to be rendered as Markdown.
   """

lib/philomena/markdown/subscript_migrator.ex

@@ -1,82 +0,0 @@
-defmodule Philomena.Markdown.SubscriptMigrator do
-  alias Philomena.Comments.Comment
-  alias Philomena.Commissions.Item, as: CommissionItem
-  alias Philomena.Commissions.Commission
-  alias Philomena.DnpEntries.DnpEntry
-  alias Philomena.Images.Image
-  alias Philomena.Conversations.Message
-  alias Philomena.ModNotes.ModNote
-  alias Philomena.Posts.Post
-  alias Philomena.Reports.Report
-  alias Philomena.Tags.Tag
-  alias Philomena.Users.User
-
-  import Ecto.Query
-  alias PhilomenaQuery.Batch
-  alias Philomena.Markdown
-  alias Philomena.Repo
-
-  @types %{
-    comments: {Comment, [:body]},
-    commission_items: {CommissionItem, [:description, :add_ons]},
-    commissions: {Commission, [:information, :contact, :will_create, :will_not_create]},
-    dnp_entries: {DnpEntry, [:conditions, :reason, :instructions]},
-    images: {Image, [:description, :scratchpad]},
-    messages: {Message, [:body]},
-    mod_notes: {ModNote, [:body]},
-    posts: {Post, [:body]},
-    reports: {Report, [:reason]},
-    tags: {Tag, [:description]},
-    users: {User, [:description, :scratchpad]}
-  }
-
-  @doc """
-  Format the ranged Markdown documents to their canonical CommonMark form.
-  """
-  @spec migrate(type :: :all | atom(), id_start :: non_neg_integer(), id_end :: non_neg_integer()) ::
-          :ok
-  def migrate(type, id_start, id_end)
-
-  def migrate(:all, _id_start, _id_end) do
-    Enum.each(@types, fn {name, _schema_columns} ->
-      migrate(name, 0, 2_147_483_647)
-    end)
-  end
-
-  def migrate(type, id_start, id_end) do
-    IO.puts("#{type}:")
-
-    {schema, columns} = Map.fetch!(@types, type)
-
-    schema
-    |> where([s], s.id >= ^id_start and s.id < ^id_end)
-    |> Batch.records()
-    |> Enum.each(fn s ->
-      case generate_updates(s, columns) do
-        [] ->
-          :ok
-
-        updates ->
-          IO.write("\r#{s.id}")
-
-          {1, nil} =
-            schema
-            |> where(id: ^s.id)
-            |> Repo.update_all(set: updates)
-      end
-    end)
-  end
-
-  @spec generate_updates(s :: struct(), columns :: [atom()]) :: Keyword.t()
-  defp generate_updates(s, columns) do
-    Enum.flat_map(columns, fn col ->
-      with value when not is_nil(value) <- Map.fetch!(s, col),
-           true <- Markdown.has_subscript?(value) do
-        [{col, Markdown.to_cm(value)}]
-      else
-        _ ->
-          []
-      end
-    end)
-  end
-end

lib/philomena/native.ex

@@ -9,12 +9,6 @@ defmodule Philomena.Native do
   @spec markdown_to_html_unsafe(String.t(), %{String.t() => String.t()}) :: String.t()
   def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded)
 
-  @spec markdown_to_cm(String.t()) :: String.t()
-  def markdown_to_cm(_text), do: :erlang.nif_error(:nif_not_loaded)
-
-  @spec markdown_has_subscript(String.t()) :: boolean()
-  def markdown_has_subscript(_text), do: :erlang.nif_error(:nif_not_loaded)
-
   @spec camo_image_url(String.t()) :: String.t()
   def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
 

lib/philomena/release.ex

@@ -14,10 +14,6 @@ defmodule Philomena.Release do
     {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
   end
 
-  def migrate_markdown(type, id_start, id_end) do
-    Philomena.Markdown.SubscriptMigrator.migrate(type, id_start, id_end)
-  end
-
   def update_channels do
     start_app()
     Philomena.Channels.update_tracked_channels!()

native/philomena/Cargo.lock

@@ -94,7 +94,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 [[package]]
 name = "comrak"
 version = "0.29.0"
-source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.1#85054b19a0383ad9c05aba1add49111c860932dc"
+source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.2#00ac2a12d5797feb0ceba9a98487451ab65593fe"
 dependencies = [
  "bon",
  "caseless",

native/philomena/Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["dylib"]
 
 [dependencies]
 base64 = "0.21"
-comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.1", default-features = false }
+comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false }
 http = "0.2"
 jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
 regex = "1"

native/philomena/src/lib.rs

@@ -15,9 +15,8 @@ static GLOBAL: Jemalloc = Jemalloc;
 rustler::init! {
     "Elixir.Philomena.Native",
     [
-        markdown_to_html, markdown_to_html_unsafe, markdown_to_cm,
+        markdown_to_html, markdown_to_html_unsafe, camo_image_url,
-        markdown_has_subscript, camo_image_url, zip_open_writer,
+        zip_open_writer, zip_start_file, zip_write, zip_finish
-        zip_start_file, zip_write, zip_finish
     ],
     load = load
 }
@@ -40,16 +39,6 @@ fn markdown_to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String
     markdown::to_html_unsafe(input, reps)
 }
 
-#[rustler::nif(schedule = "DirtyCpu")]
-fn markdown_to_cm(input: &str) -> String {
-    markdown::to_cm(input)
-}
-
-#[rustler::nif(schedule = "DirtyCpu")]
-fn markdown_has_subscript(input: &str) -> bool {
-    markdown::has_subscript(input)
-}
-
 // Camo NIF wrappers.
 
 #[rustler::nif]

native/philomena/src/markdown.rs

@@ -1,7 +1,6 @@
 use crate::{camo, domains};
-use comrak::nodes::AstNode;
+use comrak::Options;
-use comrak::{Arena, Options};
+use std::collections::HashMap;
-use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 
 pub fn common_options() -> Options {
@@ -24,7 +23,6 @@ pub fn common_options() -> Options {
     options.extension.greentext = true;
     options.extension.subscript = true;
     options.extension.philomena = true;
-    options.extension.alternate_subscript = true;
     options.render.ignore_empty_links = true;
     options.render.ignore_setext = true;
 
@@ -54,34 +52,3 @@ pub fn to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String {
 
     comrak::markdown_to_html(input, &options)
 }
-
-fn migration_options() -> Options {
-    let mut options = common_options();
-    options.extension.subscript = false;
-    options
-}
-
-pub fn to_cm(input: &str) -> String {
-    comrak::markdown_to_commonmark(input, &migration_options())
-}
-
-pub fn has_subscript(input: &str) -> bool {
-    let mut queue: VecDeque<&AstNode> = VecDeque::new();
-    let arena = Arena::new();
-
-    queue.push_back(comrak::parse_document(&arena, input, &migration_options()));
-
-    while let Some(front) = queue.pop_front() {
-        match &front.data.borrow().value {
-            comrak::nodes::NodeValue::Subscript => return true,
-            comrak::nodes::NodeValue::Strikethrough => return true,
-            _ => {}
-        }
-
-        for child in front.children() {
-            queue.push_back(child);
-        }
-    }
-
-    false
-}

native/philomena/src/tests.rs

@@ -45,7 +45,7 @@ fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) {
 
 #[test]
 fn subscript() {
-    html("H%2%O\n", "<div class=\"paragraph\">H<sub>2</sub>O</div>\n");
+    html("H~2~O\n", "<div class=\"paragraph\">H<sub>2</sub>O</div>\n");
 }
 
 #[test]
@@ -56,6 +56,14 @@ fn subscript_autolink_interaction() {
     );
 }
 
+#[test]
+fn underscore_autolink_interaction() {
+    html(
+        "https://example.com/x_",
+        "<div class=\"paragraph\"><a href=\"https://example.com/x_\">https://example.com/x_</a></div>\n"
+    )
+}
+
 #[test]
 fn spoiler() {
     html(