Update to Philomena comrak v0.29.0

2024-11-23 12:08:00 +01:00 · 2024-11-09 00:11:08 -05:00 · 2024-11-09 00:11:08 -05:00 · a21189ae10
commit a21189ae10
parent ba66c93b45
7 changed files with 356 additions and 49 deletions
--- a/.github/workflows/elixir.yml
+++ b/.github/workflows/elixir.yml
@ -68,6 +68,9 @@ jobs:
      - name: cargo clippy
        run: (cd native/philomena && cargo clippy -- -D warnings)

+      - name: cargo test
+        run: (cd native/philomena && cargo test)
+
  lint-and-test:
    name: 'JavaScript Linting and Unit Tests'
    runs-on: ubuntu-latest
--- a/native/philomena/Cargo.lock
+++ b/native/philomena/Cargo.lock
@ -32,6 +32,29 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"

+[[package]]
+name = "bon"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869"
+dependencies = [
+ "bon-macros",
+ "rustversion",
+]
+
+[[package]]
+name = "bon-macros"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663"
+dependencies = [
+ "darling",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.16.0"
@ -44,6 +67,16 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"

+[[package]]
+name = "caseless"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
+dependencies = [
+ "regex",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.94"
@ -58,12 +91,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "comrak"
-version = "0.24.1"
-source = "git+https://github.com/philomena-dev/comrak?branch=main#6a03dabfc80033b24070dc5826c9225686e3a98a"
+version = "0.29.0"
+source = "git+https://github.com/philomena-dev/comrak?branch=philomena-0.29.0#0c6fb51a55dddfc1835ed2bedfe3bcb20fb9627e"
 dependencies = [
- "derive_builder",
+ "bon",
+ "caseless",
 "entities",
- "http",
 "memchr",
 "once_cell",
 "regex",
@ -89,9 +122,9 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"

 [[package]]
 name = "darling"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
 "darling_core",
 "darling_macro",
@ -99,9 +132,9 @@ dependencies = [

 [[package]]
 name = "darling_core"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
 "fnv",
 "ident_case",
@ -113,9 +146,9 @@ dependencies = [

 [[package]]
 name = "darling_macro"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
 "darling_core",
 "quote",
@ -133,37 +166,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "derive_builder"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
-dependencies = [
- "derive_builder_macro",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
-dependencies = [
- "darling",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "derive_builder_macro"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
-dependencies = [
- "derive_builder_core",
- "syn",
-]
-
 [[package]]
 name = "deunicode"
 version = "1.4.4"
@ -463,6 +465,12 @@ dependencies = [
 "unreachable",
 ]

+[[package]]
+name = "rustversion"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
+
 [[package]]
 name = "simd-adler32"
 version = "0.3.7"
--- a/native/philomena/Cargo.toml
+++ b/native/philomena/Cargo.toml
@ -11,7 +11,7 @@ crate-type = ["dylib"]

 [dependencies]
 base64 = "0.21"
-comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main", default-features = false }
+comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.0", default-features = false }
 http = "0.2"
 jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
 regex = "1"
--- a/native/philomena/src/domains.rs
+++ b/native/philomena/src/domains.rs
@ -1,3 +1,5 @@
+use http::Uri;
+use regex::Regex;
 use std::env;

 pub fn get() -> Option<Vec<String>> {
@ -12,3 +14,21 @@ pub fn get() -> Option<Vec<String>> {

    None
 }
+
+pub fn relativize(domains: &[String], url: &str) -> Option<String> {
+    let uri = url.parse::<Uri>().ok()?;
+
+    if let Some(a) = uri.authority() {
+        if domains.contains(&a.host().to_string()) {
+            if let Ok(re) = Regex::new(&format!(r#"^http(s)?://({})"#, regex::escape(a.host()))) {
+                return Some(re.replace(url, "").into());
+            }
+        }
+    }
+
+    Some(url.into())
+}
+
+pub fn relativize_careful(domains: &[String], url: &str) -> String {
+    relativize(domains, url).unwrap_or_else(|| url.into())
+}
--- a/native/philomena/src/lib.rs
+++ b/native/philomena/src/lib.rs
@ -5,6 +5,8 @@ use std::collections::HashMap;
 mod camo;
 mod domains;
 mod markdown;
+#[cfg(test)]
+mod tests;
 mod zip;

 #[global_allocator]
--- a/native/philomena/src/markdown.rs
+++ b/native/philomena/src/markdown.rs
@ -1,37 +1,54 @@
 use crate::{camo, domains};
 use comrak::Options;
 use std::collections::HashMap;
+use std::sync::Arc;

-fn common_options() -> Options {
+pub fn common_options() -> Options {
    let mut options = Options::default();
+
+    // Upstream options
    options.extension.autolink = true;
    options.extension.table = true;
    options.extension.description_lists = true;
    options.extension.superscript = true;
    options.extension.strikethrough = true;
-    options.extension.philomena = true;
    options.parse.smart = true;
    options.render.hardbreaks = true;
    options.render.github_pre_lang = true;
+    options.render.escape = true;

-    options.extension.camoifier = Some(|s| camo::image_url_careful(&s));
-    options.extension.philomena_domains = domains::get();
+    // Philomena options
+    options.extension.underline = true;
+    options.extension.spoiler = true;
+    options.extension.greentext = true;
+    options.extension.subscript = true;
+    options.extension.philomena = true;
+    options.render.ignore_empty_links = true;
+    options.render.ignore_setext = true;
+
+    options.extension.image_url_rewriter = Some(Arc::new(|url: &str| camo::image_url_careful(url)));
+
+    if let Some(domains) = domains::get() {
+        options.extension.link_url_rewriter = Some(Arc::new(move |url: &str| {
+            domains::relativize_careful(&domains, url)
+        }));
+    }

    options
 }

 pub fn to_html(input: &str, reps: HashMap<String, String>) -> String {
    let mut options = common_options();
-    options.render.escape = true;
-    options.extension.philomena_replacements = Some(reps);
+    options.extension.replacements = Some(reps);

    comrak::markdown_to_html(input, &options)
 }

 pub fn to_html_unsafe(input: &str, reps: HashMap<String, String>) -> String {
    let mut options = common_options();
+    options.render.escape = false;
    options.render.unsafe_ = true;
-    options.extension.philomena_replacements = Some(reps);
+    options.extension.replacements = Some(reps);

    comrak::markdown_to_html(input, &options)
 }
--- a/native/philomena/src/tests.rs
+++ b/native/philomena/src/tests.rs
@ -0,0 +1,257 @@
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{domains, markdown::*};
+
+fn test_options() -> comrak::Options {
+    let mut options = common_options();
+    options.extension.image_url_rewriter = None;
+    options.extension.link_url_rewriter = None;
+    options
+}
+
+fn html(input: &str, expected: &str) {
+    html_opts_w(input, expected, &test_options());
+}
+
+fn html_opts_i<F>(input: &str, expected: &str, opts: F)
+where
+    F: Fn(&mut comrak::Options),
+{
+    let mut options = test_options();
+    opts(&mut options);
+
+    html_opts_w(input, expected, &options);
+}
+
+fn html_opts_w(input: &str, expected: &str, options: &comrak::Options) {
+    let output = comrak::markdown_to_html(input, options);
+
+    if output != expected {
+        println!("Input:");
+        println!("========================");
+        println!("{}", input);
+        println!("========================");
+        println!("Expected:");
+        println!("========================");
+        println!("{}", expected);
+        println!("========================");
+        println!("Output:");
+        println!("========================");
+        println!("{}", output);
+        println!("========================");
+    }
+    assert_eq!(output, expected);
+}
+
+#[test]
+fn subscript() {
+    html("H%2%O\n", "<div class=\"paragraph\">H<sub>2</sub>O</div>\n");
+}
+
+#[test]
+fn spoiler() {
+    html(
+        "The ||dog dies at the end of Marley and Me||.\n",
+        "<div class=\"paragraph\">The <span class=\"spoiler\">dog dies at the end of Marley and Me</span>.</div>\n",
+    );
+}
+
+#[test]
+fn spoiler_in_table() {
+    html(
+        "Text | Result\n--- | ---\n`||some clever text||` | ||some clever text||\n",
+        concat!(
+            "<table>\n",
+            "<thead>\n",
+            "<tr>\n",
+            "<th>Text</th>\n",
+            "<th>Result</th>\n",
+            "</tr>\n",
+            "</thead>\n",
+            "<tbody>\n",
+            "<tr>\n",
+            "<td><code>||some clever text||</code></td>\n",
+            "<td><span class=\"spoiler\">some clever text</span></td>\n",
+            "</tr>\n",
+            "</tbody>\n",
+            "</table>\n"
+        ),
+    );
+}
+
+#[test]
+fn spoiler_regressions() {
+    html(
+        "|should not be spoiler|\n||should be spoiler||\n|||should be spoiler surrounded by pipes|||",
+        concat!(
+            "<div class=\"paragraph\">|should not be spoiler|<br />\n",
+            "<span class=\"spoiler\">should be spoiler</span><br />\n",
+            "|<span class=\"spoiler\">should be spoiler surrounded by pipes</span>|</div>\n"
+        ),
+    );
+}
+
+#[test]
+fn mismatched_spoilers() {
+    html(
+        "|||this is a spoiler with pipe in front||\n||this is not a spoiler|\n||this is a spoiler with pipe after|||",
+        concat!(
+            "<div class=\"paragraph\">|<span class=\"spoiler\">this is a spoiler with pipe in front</span><br />\n",
+            "||this is not a spoiler|<br />\n",
+            "<span class=\"spoiler\">this is a spoiler with pipe after</span>|</div>\n"
+        ),
+    );
+}
+
+#[test]
+fn underline() {
+    html(
+        "__underlined__\n",
+        "<div class=\"paragraph\"><u>underlined</u></div>\n",
+    );
+}
+
+#[test]
+fn no_setext_headings_in_philomena() {
+    html(
+        "text text\n---",
+        "<div class=\"paragraph\">text text</div>\n<hr />\n",
+    );
+}
+
+#[test]
+fn greentext_preserved() {
+    html(
+        ">implying\n>>implying",
+        "<div class=\"paragraph\">&gt;implying<br />\n&gt;&gt;implying</div>\n",
+    );
+}
+
+#[test]
+fn separate_quotes_on_line_end() {
+    html(
+        "> 1\n>\n> 2",
+        "<blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n<div class=\"paragraph\">&gt;</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n"
+    );
+}
+
+#[test]
+fn unnest_quotes_on_line_end() {
+    html(
+        "> 1\n> > 2\n> 1",
+        "<blockquote>\n<div class=\"paragraph\">1</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n",
+    );
+}
+
+#[test]
+fn unnest_quotes_on_line_end_commonmark() {
+    html(
+        "> 1\n> > 2\n> \n> 1",
+        "<blockquote>\n<div class=\"paragraph\">1</div>\n<blockquote>\n<div class=\"paragraph\">2</div>\n</blockquote>\n<div class=\"paragraph\">1</div>\n</blockquote>\n",
+    );
+}
+
+#[test]
+fn philomena_images() {
+    html(
+        "![full](http://example.com/image.png)",
+        "<div class=\"paragraph\"><span class=\"imgspoiler\"><img src=\"http://example.com/image.png\" alt=\"full\" /></span></div>\n",
+    );
+}
+
+#[test]
+fn no_empty_link() {
+    html_opts_i(
+        "[](https://example.com/evil.domain.for.seo.spam)",
+        "<div class=\"paragraph\">[](https://example.com/evil.domain.for.seo.spam)</div>\n",
+        |opts| opts.extension.autolink = false,
+    );
+
+    html_opts_i(
+        "[    ](https://example.com/evil.domain.for.seo.spam)",
+        "<div class=\"paragraph\">[    ](https://example.com/evil.domain.for.seo.spam)</div>\n",
+        |opts| opts.extension.autolink = false,
+    );
+}
+
+#[test]
+fn empty_image_allowed() {
+    html(
+        "![   ](https://example.com/evil.domain.for.seo.spam)",
+        "<div class=\"paragraph\"><span class=\"imgspoiler\"><img src=\"https://example.com/evil.domain.for.seo.spam\" alt=\"   \" /></span></div>\n",
+    );
+}
+
+#[test]
+fn image_inside_link_allowed() {
+    html(
+        "[![](https://example.com/image.png)](https://example.com/)",
+        "<div class=\"paragraph\"><a href=\"https://example.com/\"><span class=\"imgspoiler\"><img src=\"https://example.com/image.png\" alt=\"\" /></span></a></div>\n",
+    );
+}
+
+#[test]
+fn image_mention() {
+    html_opts_i(
+        "hello world >>1234p >>1337",
+        "<div class=\"paragraph\">hello world <div id=\"1234\">p</div> &gt;&gt;1337</div>\n",
+        |opts| {
+            let mut replacements = HashMap::new();
+            replacements.insert("1234p".to_string(), "<div id=\"1234\">p</div>".to_string());
+
+            opts.extension.replacements = Some(replacements);
+        },
+    );
+}
+
+#[test]
+fn image_mention_line_start() {
+    html_opts_i(
+        ">>1234p",
+        "<div class=\"paragraph\"><div id=\"1234\">p</div></div>\n",
+        |opts| {
+            let mut replacements = HashMap::new();
+            replacements.insert("1234p".to_string(), "<div id=\"1234\">p</div>".to_string());
+
+            opts.extension.replacements = Some(replacements);
+        },
+    );
+}
+
+#[test]
+fn auto_relative_links() {
+    let domains: Vec<String> = vec!["example.com".into()];
+    let f = Arc::new(move |url: &str| domains::relativize_careful(&domains, url));
+
+    html_opts_i(
+        "[some link text](https://example.com/some/path)",
+        "<div class=\"paragraph\"><a href=\"/some/path\">some link text</a></div>\n",
+        |opts| {
+            opts.extension.link_url_rewriter = Some(f.clone());
+        },
+    );
+
+    html_opts_i(
+        "https://example.com/some/path",
+        "<div class=\"paragraph\"><a href=\"/some/path\">https://example.com/some/path</a></div>\n",
+        |opts| {
+            opts.extension.link_url_rewriter = Some(f.clone());
+        },
+    );
+
+    html_opts_i(
+        "[some link text](https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345)",
+        "<div class=\"paragraph\"><a href=\"/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345\">some link text</a></div>\n",
+        |opts| {
+            opts.extension.link_url_rewriter = Some(f.clone());
+        },
+    );
+
+    html_opts_i(
+        "https://example.com/some/path?parameter=aaaaaa&other_parameter=bbbbbb#id12345",
+        "<div class=\"paragraph\"><a href=\"/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345\">https://example.com/some/path?parameter=aaaaaa&amp;other_parameter=bbbbbb#id12345</a></div>\n",
+        |opts| {
+            opts.extension.link_url_rewriter = Some(f.clone());
+        },
+    );
+}