From c259494c1384fe2eaa37b1ac2616d594be4f69f8 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sat, 11 Sep 2021 02:37:13 +0200 Subject: [PATCH 01/25] markdown test --- .gitignore | 3 + config/config.exs | 5 + docker-compose.yml | 2 + docker/app/Dockerfile | 3 +- lib/philomena/markdown.ex | 7 + lib/philomena_web/textile_renderer.ex | 56 +-- mix.exs | 6 +- mix.lock | 3 + native/furbooru_markdown/.cargo/config | 5 + native/furbooru_markdown/Cargo.lock | 521 +++++++++++++++++++++++++ native/furbooru_markdown/Cargo.toml | 22 ++ native/furbooru_markdown/README.md | 22 ++ native/furbooru_markdown/src/lib.rs | 90 +++++ 13 files changed, 716 insertions(+), 29 deletions(-) create mode 100644 lib/philomena/markdown.ex create mode 100644 native/furbooru_markdown/.cargo/config create mode 100644 native/furbooru_markdown/Cargo.lock create mode 100644 native/furbooru_markdown/Cargo.toml create mode 100644 native/furbooru_markdown/README.md create mode 100644 native/furbooru_markdown/src/lib.rs diff --git a/.gitignore b/.gitignore index 2fa39286..fcadf67e 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ npm-debug.log # Sobelow .sobelow + +# Unportable compiled binaries +/priv/native/ diff --git a/config/config.exs b/config/config.exs index d14d544d..7df24019 100644 --- a/config/config.exs +++ b/config/config.exs @@ -36,6 +36,11 @@ config :philomena, PhilomenaWeb.Endpoint, render_errors: [view: PhilomenaWeb.ErrorView, accepts: ~w(html json)], pubsub_server: Philomena.PubSub +# Markdown +config :philomena, Philomena.Markdown, + crate: "furbooru_markdown", + mode: :release + config :phoenix, :template_engines, slim: PhoenixSlime.Engine, slime: PhoenixSlime.Engine, diff --git a/docker-compose.yml b/docker-compose.yml index 266754d6..4ac96e4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,3 +82,5 @@ services: - app ports: - '8080:80' + environment: + - RUST_LOG=trace diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 85c68a39..a53d0791 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,7 +1,7 @@ FROM elixir:1.12.0-alpine RUN apk update \ - && apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg imagemagick postgresql-client wget \ + && apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg imagemagick postgresql-client wget rust cargo \ && mix local.hex --force \ && mix local.rebar --force @@ -19,4 +19,5 @@ COPY docker/app/run-development /usr/local/bin/run-development COPY docker/app/run-test /usr/local/bin/run-test COPY docker/app/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert COPY docker/app/purge-cache /usr/local/bin/purge-cache +ENV PATH=$PATH:/root/.cargo/bin CMD run-development diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex new file mode 100644 index 00000000..b0b0cb3c --- /dev/null +++ b/lib/philomena/markdown.ex @@ -0,0 +1,7 @@ +defmodule Philomena.Markdown do + use Rustler, otp_app: :philomena + + # When your NIF is loaded, it will override this function. + def to_html(_text), do: :erlang.nif_error(:nif_not_loaded) + def to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) +end diff --git a/lib/philomena_web/textile_renderer.ex b/lib/philomena_web/textile_renderer.ex index 70e901d2..d2ef7671 100644 --- a/lib/philomena_web/textile_renderer.ex +++ b/lib/philomena_web/textile_renderer.ex @@ -14,37 +14,39 @@ defmodule PhilomenaWeb.TextileRenderer do end def render_collection(posts, conn) do - opts = %{image_transform: &Camo.Image.image_url/1} - parsed = Enum.map(posts, &Parser.parse(opts, &1.body)) + # opts = %{image_transform: &Camo.Image.image_url/1} + # parsed = Enum.map(posts, &Parser.parse(opts, &1.body)) - images = - parsed - |> Enum.flat_map(fn tree -> - tree - |> Enum.flat_map(fn - {:text, text} -> - [text] + Enum.map(posts, &Philomena.Markdown.to_html(&1.body)) - _ -> - [] - end) - end) - |> find_images + # images = + # parsed + # |> Enum.flat_map(fn tree -> + # tree + # |> Enum.flat_map(fn + # {:text, text} -> + # [text] - parsed - |> Enum.map(fn tree -> - tree - |> Enum.map(fn - {:text, text} -> - text - |> replacement_entities() - |> replacement_images(conn, images) + # _ -> + # [] + # end) + # end) + # |> find_images - {_k, markup} -> - markup - end) - |> Enum.join() - end) + # parsed + # |> Enum.map(fn tree -> + # tree + # |> Enum.map(fn + # {:text, text} -> + # text + # |> replacement_entities() + # |> replacement_images(conn, images) + + # {_k, markup} -> + # markup + # end) + # |> Enum.join() + # end) end defp replacement_entities(t) do diff --git a/mix.exs b/mix.exs index c42f8775..2b5f1299 100644 --- a/mix.exs +++ b/mix.exs @@ -11,7 +11,8 @@ defmodule Philomena.MixProject do start_permanent: Mix.env() == :prod, aliases: aliases(), deps: deps(), - dialyzer: [plt_add_apps: [:mix]] + dialyzer: [plt_add_apps: [:mix]], + rustler_crates: [furbooru_markdown: []] ] end @@ -69,6 +70,9 @@ defmodule Philomena.MixProject do {:mint, "~> 1.2"}, {:exq, "~> 0.14"}, + # Markdown + {:rustler, "~> 0.22"}, + # Linting {:credo, "~> 1.5", only: [:dev, :test], override: true}, {:credo_envvar, "~> 0.1", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 05aef432..e51e9701 100644 --- a/mix.lock +++ b/mix.lock @@ -8,6 +8,7 @@ "canary": {:hex, :canary, "1.1.1", "4138d5e05db8497c477e4af73902eb9ae06e49dceaa13c2dd9f0b55525ded48b", [:mix], [{:canada, "~> 1.0.1", [hex: :canada, repo: "hexpm", optional: false]}, {:ecto, ">= 1.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f348d9848693c830a65b707bba9e4dfdd6434e8c356a8d4477e4535afb0d653b"}, "castore": {:hex, :castore, "0.1.11", "c0665858e0e1c3e8c27178e73dffea699a5b28eb72239a3b2642d208e8594914", [:mix], [], "hexpm", "91b009ba61973b532b84f7c09ce441cba7aa15cb8b006cf06c6f4bba18220081"}, "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"}, + "cmark": {:hex, :cmark, "0.10.0", "51217dc90fec459f34a30ea73345e6a7c1d2f3d618cb4a1738a2ebd0697a57a0", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "00abeadc6f3176e3941853122413082de95d57787777dd0400d64e568acf7c32"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, @@ -64,6 +65,7 @@ "redix": {:hex, :redix, "0.10.7", "758916c71fc09e223e18d6715344581d7768c430983dabf77e792ba2087729e6", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73fdf73c0472278dc040dcd1a5da91d4febe218201ae8ac0454b37e136886c34"}, "remote_ip": {:hex, :remote_ip, "0.2.1", "cd27cd8ea54ecaaf3532776ff4c5e353b3804e710302e88c01eadeaaf42e7e24", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:inet_cidr, "~> 1.0", [hex: :inet_cidr, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2e7ab1a461cc3cd5719f37e116a08f45c8b8493923063631b164315d6b7ee8e0"}, "retry": {:hex, :retry, "0.14.1", "722d1b0cf87096b71213f5801d99fface7ca76adc83fc9dbf3e1daee952aef10", [:mix], [], "hexpm", "b3a609f286f6fe4f6b2c15f32cd4a8a60427d78d05d7b68c2dd9110981111ae0"}, + "rustler": {:hex, :rustler, "0.22.0", "e2930f9d6933e910f87526bb0a7f904e32b62a7e838a3ca4a884ee7fdfb957ed", [:mix], [{:toml, "~> 0.5.2", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "01f5989dd511ebec09be481e07d3c59773d5373c5061e09d3ebc3ef61811b49d"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, "secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"}, @@ -72,6 +74,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "tesla": {:hex, :tesla, "1.4.3", "f5a494e08fb1abe4fd9c28abb17f3d9b62b8f6fc492860baa91efb1aab61c8a0", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "e0755bb664bf4d664af72931f320c97adbf89da4586670f4864bf259b5750386"}, + "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm", "f1e3dabef71fb510d015fad18c0e05e7c57281001141504c6b69d94e99750a07"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"}, "yaml_elixir": {:hex, :yaml_elixir, "2.4.0", "2f444abc3c994c902851fde56b6a9cb82895c291c05a0490a289035c2e62ae71", [:mix], [{:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "4e25a6d5c873e393689c6f1062c5ec90f6cd1be2527b073178ae37eae4c78bee"}, diff --git a/native/furbooru_markdown/.cargo/config b/native/furbooru_markdown/.cargo/config new file mode 100644 index 00000000..a125291f --- /dev/null +++ b/native/furbooru_markdown/.cargo/config @@ -0,0 +1,5 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/native/furbooru_markdown/Cargo.lock b/native/furbooru_markdown/Cargo.lock new file mode 100644 index 00000000..369437fc --- /dev/null +++ b/native/furbooru_markdown/Cargo.lock @@ -0,0 +1,521 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clap" +version = "2.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "comrak" +version = "0.7.0" +source = "git+https://github.com/furbooru/comrak#ee99f4fad1d0178ad826201a9eff91cc13b4f0ef" +dependencies = [ + "clap", + "entities", + "lazy_static", + "log", + "pest", + "pest_derive", + "regex", + "twoway", + "typed-arena", + "unicode_categories", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "furbooru_markdown" +version = "0.1.1" +dependencies = [ + "comrak", + "lazy_static", + "log", + "pretty_env_logger", + "regex", + "rustler", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger", + "log", +] + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "rustler" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b787d3b2a80007f41cd4c0c310cdeb3936192768159585f65ecc7e96faf97fc3" +dependencies = [ + "lazy_static", + "rustler_codegen", + "rustler_sys", +] + +[[package]] +name = "rustler_codegen" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a1f867002b6f0130f47abf215cac4405646db6f5d7b009b21c890980490aa4" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustler_sys" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb96034ff33723615fd19223d58c987c1f6476342e83557a6e467ef95f83bda" +dependencies = [ + "unreachable", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "twoway" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b40075910de3a912adbd80b5d8bad6ad10a23eeb1f5bf9d4006839e899ba5bc" +dependencies = [ + "memchr", + "unchecked-index", +] + +[[package]] +name = "typed-arena" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/native/furbooru_markdown/Cargo.toml b/native/furbooru_markdown/Cargo.toml new file mode 100644 index 00000000..9cbc1d94 --- /dev/null +++ b/native/furbooru_markdown/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "furbooru_markdown" +version = "0.1.1" +authors = [] +edition = "2018" + +[lib] +name = "furbooru_markdown" +path = "src/lib.rs" +crate-type = ["dylib"] + +[dependencies] +comrak = { git = "https://github.com/furbooru/comrak" } +lazy_static = "1.0" +regex = "1" +log = "0" +pretty_env_logger = "0" +rustler = "0.22.0" + +[profile.release] +opt-level = 3 +lto = true diff --git a/native/furbooru_markdown/README.md b/native/furbooru_markdown/README.md new file mode 100644 index 00000000..e884ccc5 --- /dev/null +++ b/native/furbooru_markdown/README.md @@ -0,0 +1,22 @@ +# NIF for Elixir.Furbooru.Markdown + +## To build the NIF module: + +- Make sure your projects `mix.exs` has the `:rustler` compiler listed in the `project` function: `compilers: [:rustler] ++ Mix.compilers()` If there already is a `:compilers` list, you should append `:rustler` to it. +- Add your crate to the `rustler_crates` attribute in the `project function. [See here](https://hexdocs.pm/rustler/basics.html#crate-configuration). +- Your NIF will now build along with your project. + +## To load the NIF: + +```elixir +defmodule Furbooru.Markdown do + use Rustler, otp_app: , crate: "furbooru_markdown" + + # When your NIF is loaded, it will override this function. + def add(_a, _b), do: :erlang.nif_error(:nif_not_loaded) +end +``` + +## Examples + +[This](https://github.com/hansihe/NifIo) is a complete example of a NIF written in Rust. diff --git a/native/furbooru_markdown/src/lib.rs b/native/furbooru_markdown/src/lib.rs new file mode 100644 index 00000000..8b36849c --- /dev/null +++ b/native/furbooru_markdown/src/lib.rs @@ -0,0 +1,90 @@ +use comrak::{markdown_to_html, ComrakOptions}; +use lazy_static::lazy_static; +use regex::{Captures, Regex}; + +mod atoms { + rustler::atoms! { + ok, + } +} + +rustler::init! { + "Elixir.Philomena.Markdown", + [to_html, to_html_unsafe] +} + +const IMAGE_MENTION_REGEX: &'static str = r#">>([0-9]+)([|t|s|p]?)"#; + +lazy_static! { + static ref IMAGE_MENTION_REPLACE: Regex = Regex::new(IMAGE_MENTION_REGEX).unwrap(); +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn to_html(input: String) -> String { + let _ = pretty_env_logger::try_init(); + let mut text: String = input; + + if text.contains(">>") { + text = text.replace(">>", ">>"); + } + + let mut options = ComrakOptions::default(); + options.extension.autolink = true; + options.extension.table = true; + options.extension.description_lists = true; + options.extension.superscript = true; + options.extension.subscript = true; + options.extension.spoiler = true; + options.extension.strikethrough = true; + // options.extension.furbooru = true; + options.parse.smart = true; + options.render.hardbreaks = true; + options.render.github_pre_lang = true; + let mut result = markdown_to_html(&text, &options); + + result = match IMAGE_MENTION_REPLACE.captures(&result) { + None => result, + Some(fields) => { + match fields.get(2).unwrap().as_str() { + "t" => result, // TODO(Xe): thumbnail rendering + "s" => result, // TODO(Xe): small preview rendering + "p" => result, // TODO(Xe): large preview rendering + "" => String::from( + IMAGE_MENTION_REPLACE.replace_all(&result, |caps: &Captures| { + format!(r#">>{0}"#, &caps[1]) + }), + ), + _ => result, + } + } + }; + + result +} + +#[rustler::nif(schedule = "DirtyCpu")] +fn to_html_unsafe(input: String) -> String { + let mut text: String = input; + + if text.contains(">>") { + text = text.replace(">>", ">>"); + } + + let mut options = ComrakOptions::default(); + options.extension.autolink = true; + options.extension.table = true; + options.extension.description_lists = true; + options.extension.superscript = true; + options.extension.subscript = true; + options.extension.spoiler = true; + options.extension.strikethrough = true; + // options.extension.furbooru = true; + options.parse.smart = true; + options.render.hardbreaks = true; + options.render.github_pre_lang = true; + options.render.unsafe_ = true; + + let result = markdown_to_html(&text, &options); + + result +} From 245eb781c83843423fdb45b3026efe4ea8971c00 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sat, 11 Sep 2021 23:21:25 +0200 Subject: [PATCH 02/25] cargo and mix updates --- mix.lock | 18 +- native/furbooru_markdown/Cargo.lock | 392 +++++++++++++++++++++++----- 2 files changed, 340 insertions(+), 70 deletions(-) diff --git a/mix.lock b/mix.lock index e51e9701..8ccf9143 100644 --- a/mix.lock +++ b/mix.lock @@ -8,11 +8,11 @@ "canary": {:hex, :canary, "1.1.1", "4138d5e05db8497c477e4af73902eb9ae06e49dceaa13c2dd9f0b55525ded48b", [:mix], [{:canada, "~> 1.0.1", [hex: :canada, repo: "hexpm", optional: false]}, {:ecto, ">= 1.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f348d9848693c830a65b707bba9e4dfdd6434e8c356a8d4477e4535afb0d653b"}, "castore": {:hex, :castore, "0.1.11", "c0665858e0e1c3e8c27178e73dffea699a5b28eb72239a3b2642d208e8594914", [:mix], [], "hexpm", "91b009ba61973b532b84f7c09ce441cba7aa15cb8b006cf06c6f4bba18220081"}, "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"}, - "cmark": {:hex, :cmark, "0.10.0", "51217dc90fec459f34a30ea73345e6a7c1d2f3d618cb4a1738a2ebd0697a57a0", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "00abeadc6f3176e3941853122413082de95d57787777dd0400d64e568acf7c32"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:hex, :credo, "1.5.6", "e04cc0fdc236fefbb578e0c04bd01a471081616e741d386909e527ac146016c6", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4b52a3e558bd64e30de62a648518a5ea2b6e3e5d2b164ef5296244753fc7eb17"}, "credo_envvar": {:hex, :credo_envvar, "0.1.4", "40817c10334e400f031012c0510bfa0d8725c19d867e4ae39cf14f2cbebc3b20", [:mix], [{:credo, "~> 1.0", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "5055cdb4bcbaf7d423bc2bb3ac62b4e2d825e2b1e816884c468dee59d0363009"}, @@ -20,9 +20,9 @@ "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, - "ecto": {:hex, :ecto, "3.7.1", "a20598862351b29f80f285b21ec5297da1181c0442687f9b8329f0445d228892", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d36e5b39fc479e654cffd4dbe1865d9716e4a9b6311faff799b6f90ab81b8638"}, + "ecto": {:hex, :ecto, "3.5.8", "8ebf12be6016cb99313348ba7bb4612f4114b9a506d6da79a2adc7ef449340bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea0be182ea8922eb7742e3ae8e71b67ee00ae177de1bf76210299a5f16ba4c77"}, "ecto_network": {:hex, :ecto_network, "1.3.0", "1e77fa37c20e0f6a426d3862732f3317b0fa4c18f123d325f81752a491d7304e", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "053a5e46ef2837e8ea5ea97c82fa0f5494699209eddd764e663c85f11b2865bd"}, - "ecto_sql": {:hex, :ecto_sql, "3.7.0", "2fcaad4ab0c8d76a5afbef078162806adbe709c04160aca58400d5cbbe8eeac6", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a26135dfa1d99bf87a928c464cfa25bba6535a4fe761eefa56077a4febc60f70"}, + "ecto_sql": {:hex, :ecto_sql, "3.5.0", "760aa2935cc80b72da83fbd8cc97923623a2401915c308afea2cf2b0aabf4b2e", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3bab456e3ebb5680b327313f57ebb5356882a59fe04964a03232a83dc4c44aa2"}, "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"}, "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, @@ -46,7 +46,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "pbkdf2": {:git, "https://github.com/code-time/erlang-pbkdf2.git", "f8f0012a97f58ade9c70ac93260e4259e4ca4b8d", [ref: "f8f0012a97f58ade9c70ac93260e4259e4ca4b8d"]}, - "phoenix": {:hex, :phoenix, "1.5.12", "75fddb14c720388eea93d33886166a690416a7ff8633fbd93f364355b6fe1166", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8f0ae6734fcc18bbaa646c161e2febc46fb899eae43f82679b92530983324113"}, + "phoenix": {:hex, :phoenix, "1.5.0", "59cf8c734a0e305736654961691aeaa11d80b96a0cc4aeb68d8610af42af1aef", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c695e372ea39914bb97d62ef632550bc3bb38a4bb78fa520f5d5d4be3630399e"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "2.14.3", "51f720d0d543e4e157ff06b65de38e13303d5778a7919bcc696599e5934271b8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "efd697a7fff35a13eeeb6b43db884705cba353a1a41d127d118fda5f90c8e80f"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.3.3", "3a53772a6118d5679bf50fc1670505a290e32a1d195df9e069d8c53ab040c054", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "766796676e5f558dbae5d1bdb066849673e956005e3730dfd5affd7a6da4abac"}, @@ -54,15 +54,15 @@ "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"}, "phoenix_pubsub_redis": {:hex, :phoenix_pubsub_redis, "3.0.1", "d4d856b1e57a21358e448543e1d091e07e83403dde4383b8be04ed9d2c201cbc", [:mix], [{:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1 or ~> 1.6", [hex: :poolboy, repo: "hexpm", optional: false]}, {:redix, "~> 0.10.0 or ~> 1.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "0b36a17ff6e9a56159f8df8933d62b5c1f0695eae995a02e0c86c035ace6a309"}, "phoenix_slime": {:hex, :phoenix_slime, "0.13.1", "a5d4d8febb87a618b02d690519f7106832c8bd0b4d1937fbba73d6e8666f2891", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:slime, "~> 1.0", [hex: :slime, repo: "hexpm", optional: false]}], "hexpm", "ff818744be2c903fb0174ba22b230c1c335238578fccf274b1d95d08f4844377"}, - "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"}, + "plug": {:hex, :plug, "1.10.0", "6508295cbeb4c654860845fb95260737e4a8838d34d115ad76cd487584e2fc4d", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "422a9727e667be1bf5ab1de03be6fa0ad67b775b2d84ed908f3264415ef29d4a"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"}, "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.15.10", "2809dee1b1d76f7cbabe570b2a9285c2e7b41be60cf792f5f2804a54b838a067", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "1560ca427542f6b213f8e281633ae1a3b31cdbcd84ebd7f50628765b8f6132be"}, "pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"}, "qrcode": {:hex, :qrcode, "0.1.5", "551271830515c150f34568345b060c625deb0e6691db2a01b0a6de3aafc93886", [:mix], [], "hexpm", "a266b7fb7be0d3b713912055dde3575927eca920e5d604ded45cd534f6b7a447"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "redix": {:hex, :redix, "0.10.7", "758916c71fc09e223e18d6715344581d7768c430983dabf77e792ba2087729e6", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73fdf73c0472278dc040dcd1a5da91d4febe218201ae8ac0454b37e136886c34"}, + "redix": {:hex, :redix, "0.10.0", "886cfbb14f9b78b82f38963695be2c6ed54b4f5cf911acbf70278ba09144f55d", [:mix], [{:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d8266f5a13291f1d8d7003ad48820e80d34c48348dd762e6564a806cb4b5bc0f"}, "remote_ip": {:hex, :remote_ip, "0.2.1", "cd27cd8ea54ecaaf3532776ff4c5e353b3804e710302e88c01eadeaaf42e7e24", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:inet_cidr, "~> 1.0", [hex: :inet_cidr, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2e7ab1a461cc3cd5719f37e116a08f45c8b8493923063631b164315d6b7ee8e0"}, "retry": {:hex, :retry, "0.14.1", "722d1b0cf87096b71213f5801d99fface7ca76adc83fc9dbf3e1daee952aef10", [:mix], [], "hexpm", "b3a609f286f6fe4f6b2c15f32cd4a8a60427d78d05d7b68c2dd9110981111ae0"}, "rustler": {:hex, :rustler, "0.22.0", "e2930f9d6933e910f87526bb0a7f904e32b62a7e838a3ca4a884ee7fdfb957ed", [:mix], [{:toml, "~> 0.5.2", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "01f5989dd511ebec09be481e07d3c59773d5373c5061e09d3ebc3ef61811b49d"}, @@ -72,8 +72,8 @@ "slime": {:hex, :slime, "1.2.1", "71e036056051f0a6fae136af34eaa1322e8e11cdd2da3a56196fd31bca34dd49", [:mix], [{:neotoma, "~> 1.7", [hex: :neotoma, repo: "hexpm", optional: false]}], "hexpm", "298568e64291fed4eb690be094f6c46400daa03b594bab34fcaa0167e139c263"}, "sobelow": {:hex, :sobelow, "0.11.1", "23438964486f8112b41e743bbfd402da3e5b296fdc9eacab29914b79c48916dd", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9897363a7eff96f4809304a90aad819e2ad5e5d24db547af502885146746a53c"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, - "tesla": {:hex, :tesla, "1.4.3", "f5a494e08fb1abe4fd9c28abb17f3d9b62b8f6fc492860baa91efb1aab61c8a0", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "e0755bb664bf4d664af72931f320c97adbf89da4586670f4864bf259b5750386"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm", "e9e3cacfd37c1531c0ca70ca7c0c30ce2dbb02998a4f7719de180fe63f8d41e4"}, + "tesla": {:hex, :tesla, "1.4.0", "1081bef0124b8bdec1c3d330bbe91956648fb008cf0d3950a369cda466a31a87", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "bf1374a5569f5fca8e641363b63f7347d680d91388880979a33bc12a6eb3e0aa"}, "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm", "f1e3dabef71fb510d015fad18c0e05e7c57281001141504c6b69d94e99750a07"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"}, diff --git a/native/furbooru_markdown/Cargo.lock b/native/furbooru_markdown/Cargo.lock index 369437fc..2a6e6bab 100644 --- a/native/furbooru_markdown/Cargo.lock +++ b/native/furbooru_markdown/Cargo.lock @@ -1,10 +1,16 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] -name = "aho-corasick" -version = "0.7.13" +name = "adler" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -30,10 +36,31 @@ dependencies = [ ] [[package]] -name = "bitflags" -version = "1.2.1" +name = "autocfg" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" @@ -64,21 +91,37 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" [[package]] name = "cfg-if" -version = "0.1.10" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "num-integer", + "num-traits", +] [[package]] name = "clap" -version = "2.33.1" +version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ "ansi_term", "atty", @@ -91,8 +134,8 @@ dependencies = [ [[package]] name = "comrak" -version = "0.7.0" -source = "git+https://github.com/furbooru/comrak#ee99f4fad1d0178ad826201a9eff91cc13b4f0ef" +version = "0.12.1" +source = "git+https://github.com/furbooru/comrak#4adb01d736b03a25c81451d62148da37256e99c7" dependencies = [ "clap", "entities", @@ -101,9 +144,21 @@ dependencies = [ "pest", "pest_derive", "regex", + "shell-words", + "syntect", "twoway", "typed-arena", "unicode_categories", + "xdg", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", ] [[package]] @@ -140,6 +195,24 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "flate2" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80edafed416a46fb378521624fab1cfa2eb514784fd8921adbe8a8d8321da811" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "furbooru_markdown" version = "0.1.1" @@ -154,27 +227,33 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" dependencies = [ "typenum", ] [[package]] -name = "heck" -version = "0.3.1" +name = "hashbrown" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] @@ -188,6 +267,22 @@ dependencies = [ "quick-error", ] +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "lazy_static" version = "1.4.0" @@ -195,16 +290,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libc" -version = "0.2.72" +name = "lazycell" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" + +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] [[package]] name = "log" -version = "0.4.8" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if", ] @@ -217,9 +327,60 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "memchr" -version = "2.3.3" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "onig" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16fd3c0e73b516af509c13c4ba76ec0c987ce20d78b38cff356b8d01fc6a6c0" +dependencies = [ + "bitflags", + "lazy_static", + "libc", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd9442a09e4fbd08d196ddf419b2c79a43c3a46c800320cc841d45c2449a240" +dependencies = [ + "cc", + "pkg-config", +] [[package]] name = "opaque-debug" @@ -270,6 +431,26 @@ dependencies = [ "sha-1", ] +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "plist" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38d026d73eeaf2ade76309d0c65db5a35ecf649e3cec428db316243ea9d6711" +dependencies = [ + "base64", + "chrono", + "indexmap", + "line-wrap", + "serde", + "xml-rs", +] + [[package]] name = "pretty_env_logger" version = "0.4.0" @@ -282,9 +463,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.18" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" dependencies = [ "unicode-xid", ] @@ -297,30 +478,29 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.3.9" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "rustler" @@ -347,13 +527,62 @@ dependencies = [ [[package]] name = "rustler_sys" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb96034ff33723615fd19223d58c987c1f6476342e83557a6e467ef95f83bda" +checksum = "1cb382fde4f421c51555919e9920b058c0286f6bf59e53d02eb4d281eae6758b" dependencies = [ "unreachable", ] +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -366,6 +595,12 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "shell-words" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074" + [[package]] name = "strsim" version = "0.8.0" @@ -374,9 +609,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.33" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84" dependencies = [ "proc-macro2", "quote", @@ -384,10 +619,31 @@ dependencies = [ ] [[package]] -name = "termcolor" -version = "1.1.0" +name = "syntect" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031" +dependencies = [ + "bincode", + "bitflags", + "flate2", + "fnv", + "lazy_static", + "lazycell", + "onig", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "walkdir", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" dependencies = [ "winapi-util", ] @@ -401,20 +657,11 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - [[package]] name = "twoway" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b40075910de3a912adbd80b5d8bad6ad10a23eeb1f5bf9d4006839e899ba5bc" +checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" dependencies = [ "memchr", "unchecked-index", @@ -428,9 +675,9 @@ checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" [[package]] name = "typenum" -version = "1.12.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "ucd-trie" @@ -446,9 +693,9 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" [[package]] name = "unicode-segmentation" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" [[package]] name = "unicode-width" @@ -458,9 +705,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "unicode_categories" @@ -489,6 +736,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "winapi" version = "0.3.9" @@ -519,3 +777,15 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xdg" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" + +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" From f9a7a28df3b3a40a90a208dd152875f5fa751bbd Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 00:47:40 +0200 Subject: [PATCH 03/25] dgr --- native/furbooru_markdown/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/native/furbooru_markdown/src/lib.rs b/native/furbooru_markdown/src/lib.rs index 8b36849c..e01b9c88 100644 --- a/native/furbooru_markdown/src/lib.rs +++ b/native/furbooru_markdown/src/lib.rs @@ -1,6 +1,7 @@ use comrak::{markdown_to_html, ComrakOptions}; use lazy_static::lazy_static; use regex::{Captures, Regex}; +use rustler::Term; mod atoms { rustler::atoms! { @@ -40,6 +41,7 @@ fn to_html(input: String) -> String { options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; + options.render.escape = true; let mut result = markdown_to_html(&text, &options); result = match IMAGE_MENTION_REPLACE.captures(&result) { @@ -51,7 +53,7 @@ fn to_html(input: String) -> String { "p" => result, // TODO(Xe): large preview rendering "" => String::from( IMAGE_MENTION_REPLACE.replace_all(&result, |caps: &Captures| { - format!(r#">>{0}"#, &caps[1]) + format!(r#">>{0}"#, &caps[1]) }), ), _ => result, @@ -78,6 +80,7 @@ fn to_html_unsafe(input: String) -> String { options.extension.subscript = true; options.extension.spoiler = true; options.extension.strikethrough = true; + options.extension.front_matter_delimiter = Some("---".to_owned()); // options.extension.furbooru = true; options.parse.smart = true; options.render.hardbreaks = true; From d2582aac02b107e507c53122b19e914bb7f10483 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Sun, 12 Sep 2021 10:27:02 -0400 Subject: [PATCH 04/25] Use jemalloc instead of the system allocator for Rust --- docker/app/run-development | 3 +++ native/furbooru_markdown/Cargo.lock | 30 +++++++++++++++++++++++++++++ native/furbooru_markdown/Cargo.toml | 1 + native/furbooru_markdown/src/lib.rs | 4 ++++ 4 files changed, 38 insertions(+) diff --git a/docker/app/run-development b/docker/app/run-development index e257ddab..529fc9ca 100755 --- a/docker/app/run-development +++ b/docker/app/run-development @@ -1,5 +1,8 @@ #!/usr/bin/env sh +# For compatibility with musl libc +export CARGO_FEATURE_DISABLE_INITIAL_EXEC_TLS=1 + background() { while :; do mix run -e 'Philomena.Release.update_channels()' diff --git a/native/furbooru_markdown/Cargo.lock b/native/furbooru_markdown/Cargo.lock index 2a6e6bab..bf632bd9 100644 --- a/native/furbooru_markdown/Cargo.lock +++ b/native/furbooru_markdown/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "1.0.2" @@ -213,11 +215,18 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + [[package]] name = "furbooru_markdown" version = "0.1.1" dependencies = [ "comrak", + "jemallocator", "lazy_static", "log", "pretty_env_logger", @@ -283,6 +292,27 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" diff --git a/native/furbooru_markdown/Cargo.toml b/native/furbooru_markdown/Cargo.toml index 9cbc1d94..7c21e352 100644 --- a/native/furbooru_markdown/Cargo.toml +++ b/native/furbooru_markdown/Cargo.toml @@ -16,6 +16,7 @@ regex = "1" log = "0" pretty_env_logger = "0" rustler = "0.22.0" +jemallocator = "0.3.2" [profile.release] opt-level = 3 diff --git a/native/furbooru_markdown/src/lib.rs b/native/furbooru_markdown/src/lib.rs index e01b9c88..7d834349 100644 --- a/native/furbooru_markdown/src/lib.rs +++ b/native/furbooru_markdown/src/lib.rs @@ -2,6 +2,10 @@ use comrak::{markdown_to_html, ComrakOptions}; use lazy_static::lazy_static; use regex::{Captures, Regex}; use rustler::Term; +use jemallocator::Jemalloc; + +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; mod atoms { rustler::atoms! { From 4b46ca803f3993e517473abc354f82e65af25199 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 16:33:05 +0200 Subject: [PATCH 05/25] preliminary textile-markdown converter --- assets/css/common/_base.scss | 20 + assets/css/common/_dimensions.scss | 5 + lib/philomena/markdown.ex | 7 + lib/philomena/textile/parser_markdown.ex | 486 +++++++++++++++++++++++ lib/philomena_web/textile_renderer.ex | 4 +- native/furbooru_markdown/src/lib.rs | 1 - 6 files changed, 520 insertions(+), 3 deletions(-) create mode 100644 lib/philomena/textile/parser_markdown.ex diff --git a/assets/css/common/_base.scss b/assets/css/common/_base.scss index 99ccfa6f..a59f2309 100644 --- a/assets/css/common/_base.scss +++ b/assets/css/common/_base.scss @@ -267,6 +267,26 @@ blockquote { margin-bottom: 6px; } +img[alt=tiny] { + max-height: $image_tiny_size; + max-width: $image_tiny_size; +} + +img[alt=small] { + max-height: $image_small_size; + max-width: $image_small_size; +} + +img[alt=medium] { + max-height: $image_medium_size; + max-width: $image_medium_size; +} + +img[alt=large] { + max-height: $image_large_size; + max-width: $image_large_size; +} + //donations .donate-button { background: 0; diff --git a/assets/css/common/_dimensions.scss b/assets/css/common/_dimensions.scss index d9a43ccf..2aab5abe 100644 --- a/assets/css/common/_dimensions.scss +++ b/assets/css/common/_dimensions.scss @@ -16,6 +16,11 @@ $medium_layout_width: 1330px; $centered_layout_side_margin: 24px; $layout_side_margin: 12px; +$image_tiny_size: 64px; +$image_small_size: 128px; +$image_medium_size: 256px; +$image_large_size: 512px; + $header_height: 36px; $header_field_height: 28px; $header_sub_height: 32px; diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index b0b0cb3c..fd8f8563 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -1,7 +1,14 @@ defmodule Philomena.Markdown do use Rustler, otp_app: :philomena + @markdown_chars = ~r/[\*_\[\]\(\)\^`\%\\~<>#\|]/ + # When your NIF is loaded, it will override this function. def to_html(_text), do: :erlang.nif_error(:nif_not_loaded) def to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) + + def escape_markdown(text) do + @markdown_chars + |> Regex.replace(text, "\\\\0") + end end diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex new file mode 100644 index 00000000..0fb47133 --- /dev/null +++ b/lib/philomena/textile/parser_markdown.ex @@ -0,0 +1,486 @@ +defmodule Philomena.Textile.ParserMarkdown do + alias Philomena.Textile.Lexer + alias Philomena.Markdown + alias Phoenix.HTML + + defp markdown_quote(text) do + result = Regexp.replace(~r/\n/, text, "\\0> ") + "> #{result}" + end + + def parse(parser, input) do + parser = Map.put(parser, :state, %{}) + + with {:ok, tokens, _1, _2, _3, _4} <- Lexer.lex(String.trim(input || "")), + {:ok, tree, []} <- repeat(&textile/2, parser, tokens) do + partial_flatten(tree) + else + _ -> + [] + end + end + + # Helper to turn a parse tree into a string + def flatten(tree) do + tree + |> List.flatten() + |> Enum.map_join("", fn {_k, v} -> v end) + end + + # Helper to escape HTML + defp escape(text) do + text + |> HTML.html_escape() + |> HTML.safe_to_string() + end + + # Helper to turn a parse tree into a list + def partial_flatten(tree) do + tree + |> List.flatten() + |> Enum.chunk_by(fn {k, _v} -> k end) + |> Enum.map(fn list -> + [{type, _v} | _rest] = list + + value = Enum.map_join(list, "", fn {_k, v} -> v end) + + {type, value} + end) + end + + defp put_state(parser, new_state) do + state = Map.put(parser.state, new_state, true) + Map.put(parser, :state, state) + end + + # Helper corresponding to Kleene star (*) operator + # Match a specificed rule zero or more times + defp repeat(rule, parser, tokens) do + case rule.(parser, tokens) do + {:ok, tree, r_tokens} -> + {:ok, tree2, r2_tokens} = repeat(rule, parser, r_tokens) + {:ok, [tree, tree2], r2_tokens} + + _ -> + {:ok, [], tokens} + end + end + + # Helper to match a simple recursive grammar rule of the following form: + # + # open_token callback* close_token + # + defp simple_recursive(open_token, close_token, open_tag, close_tag, callback, parser, [ + {open_token, open} | r_tokens + ]) do + case repeat(callback, parser, r_tokens) do + {:ok, tree, [{^close_token, _} | r2_tokens]} -> + {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + + defp simple_recursive( + _open_token, + _close_token, + _open_tag, + _close_tag, + _callback, + _parser, + _tokens + ) do + {:error, "Expected a simple recursive rule"} + end + + # Helper to match a simple recursive grammar rule with negative lookahead: + # + # open_token callback* close_token (?!lookahead_not) + # + defp simple_lookahead_not( + open_token, + close_token, + open_tag, + close_tag, + lookahead_not, + callback, + state, + parser, + [{open_token, open} | r_tokens] + ) do + case parser.state do + %{^state => _} -> + {:error, "End of rule"} + + _ -> + case r_tokens do + [{forbidden_lookahead, _la} | _] when forbidden_lookahead in [:space, :newline] -> + {:ok, [{:text, escape(open)}], r_tokens} + + _ -> + case repeat(callback, put_state(parser, state), r_tokens) do + {:ok, tree, [{^close_token, close}, {^lookahead_not, ln} | r2_tokens]} -> + {:ok, [{:text, escape(open)}, tree, {:text, escape(close)}], + [{lookahead_not, ln} | r2_tokens]} + + {:ok, tree, [{^close_token, _} | r2_tokens]} -> + {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + end + end + + defp simple_lookahead_not( + _open_token, + _close_token, + _open_tag, + _close_tag, + _lookahead_not, + _callback, + _state, + _parser, + _tokens + ) do + {:error, "Expected a simple lookahead not rule"} + end + + # Helper to efficiently assemble a UTF-8 binary from tokens of the + # given type + defp assemble_binary(token_type, accumulator, [{token_type, t} | stream]) do + assemble_binary(token_type, accumulator <> <>, stream) + end + + defp assemble_binary(_token_type, accumulator, tokens), do: {accumulator, tokens} + + # + # inline_textile_element = + # opening_markup inline_textile_element* closing_markup (?!quicktxt) | + # closing_markup (?=quicktxt) | + # link_delim block_textile_element* link_url | + # image url? | + # code_delim inline_textile_element* code_delim | + # inline_textile_element_not_opening_markup; + # + + defp inline_textile_element(parser, tokens) do + [ + {:b_delim, :b, "**", "**"}, + {:i_delim, :i, "_", "_"}, + {:strong_delim, :strong, "**", "**"}, + {:em_delim, :em, "*", "*"}, + {:ins_delim, :ins, "__", "__"}, + {:sup_delim, :sup, "^", "^"}, + {:del_delim, :del, "~~", "~~"}, + {:sub_delim, :sub, "%", "%"} + ] + |> Enum.find_value(fn {delim_token, state, open_tag, close_tag} -> + simple_lookahead_not( + delim_token, + delim_token, + open_tag, + close_tag, + :quicktxt, + &inline_textile_element/2, + state, + parser, + tokens + ) + |> case do + {:ok, tree, r_tokens} -> + {:ok, tree, r_tokens} + + _ -> + nil + end + end) + |> case do + nil -> inner_inline_textile_element(parser, tokens) + value -> value + end + end + + defp inner_inline_textile_element(parser, [{token, t}, {:quicktxt, q} | r_tokens]) + when token in [ + :b_delim, + :i_delim, + :strong_delim, + :em_delim, + :ins_delim, + :sup_delim, + :del_delim, + :sub_delim + ] do + case inline_textile_element(parser, [{:quicktxt, q} | r_tokens]) do + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(t)}, tree], r2_tokens} + + _ -> + {:ok, [{:text, escape(t)}], [{:quicktxt, q} | r_tokens]} + end + end + + defp inner_inline_textile_element(parser, [{:link_delim, open} | r_tokens]) do + case repeat(&block_textile_element/2, parser, r_tokens) do + {:ok, tree, [{:unbracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> + href = escape(url) + + {:ok, + [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], + r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + + defp inner_inline_textile_element(parser, [{:bracketed_link_open, open} | r_tokens]) do + case repeat(&inline_textile_element/2, parser, r_tokens) do + {:ok, tree, [{:bracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> + href = escape(url) + + {:ok, + [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], + r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + + defp inner_inline_textile_element(parser, [ + {token, img}, + {:unbracketed_image_url, <<":", url::binary>>} | r_tokens + ]) + when token in [:unbracketed_image, :bracketed_image] do + img = parser.image_transform.(img) + + {:ok, + [ + {:markup, "[![full]("}, + {:markup, escape(img)}, + {:markup, ")]("}, + {:markup, escape(url)}, + {:markup, ")"} + ], r_tokens} + end + + defp inner_inline_textile_element(parser, [{token, img} | r_tokens]) + when token in [:unbracketed_image, :bracketed_image] do + img = parser.image_transform.(img) + + {:ok, + [ + {:markup, "![full]("}, + {:markup, escape(img)}, + {:markup, ")"} + ], r_tokens} + end + + defp inner_inline_textile_element(parser, [{:code_delim, open} | r_tokens]) do + case parser.state do + %{code: _} -> + {:error, "End of rule"} + + _ -> + case repeat(&inline_textile_element/2, put_state(parser, :code), r_tokens) do + {:ok, tree, [{:code_delim, _} | r2_tokens]} -> + {:ok, [{:markup, "`"}, tree, {:markup, "`"}], r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + end + + defp inner_inline_textile_element(parser, tokens) do + inline_textile_element_not_opening_markup(parser, tokens) + end + + # + # bq_cite_text = (?!bq_cite_open); + # + + # Note that text is not escaped here because it will be escaped + # when the tree is flattened + defp bq_cite_text(_parser, [{:bq_cite_open, _open} | _rest]) do + {:error, "Expected cite tokens"} + end + + defp bq_cite_text(_parser, [{:char, lit} | r_tokens]) do + {:ok, [{:text, <>}], r_tokens} + end + + defp bq_cite_text(_parser, [{:quicktxt, lit} | r_tokens]) do + {:ok, [{:text, <>}], r_tokens} + end + + defp bq_cite_text(_parser, [{:space, _} | r_tokens]) do + {:ok, [{:text, " "}], r_tokens} + end + + defp bq_cite_text(_parser, [{_token, t} | r_tokens]) do + {:ok, [{:text, t}], r_tokens} + end + + defp bq_cite_text(_parser, _tokens) do + {:error, "Expected cite tokens"} + end + + # + # inline_textile_element_not_opening_markup = + # literal | space | char | + # quicktxt opening_markup quicktxt | + # quicktxt | + # opening_block_tag block_textile_element* closing_block_tag; + # + + defp inline_textile_element_not_opening_markup(_parser, [{:literal, lit} | r_tokens]) do + {:ok, [{:markup, escape(Markdown.escape_markdown(lit))},], + r_tokens} + end + + defp inline_textile_element_not_opening_markup(_parser, [{:space, _} | r_tokens]) do + {:ok, [{:text, " "}], r_tokens} + end + + defp inline_textile_element_not_opening_markup(_parser, [{:char, lit} | r_tokens]) do + {binary, r2_tokens} = assemble_binary(:char, <>, r_tokens) + + {:ok, [{:text, escape(binary)}], r2_tokens} + end + + defp inline_textile_element_not_opening_markup(_parser, [ + {:quicktxt, q1}, + {token, t}, + {:quicktxt, q2} | r_tokens + ]) + when token in [ + :b_delim, + :i_delim, + :strong_delim, + :em_delim, + :ins_delim, + :sup_delim, + :del_delim, + :sub_delim + ] do + {:ok, [{:text, escape(<>)}, {:text, escape(t)}, {:text, escape(<>)}], + r_tokens} + end + + defp inline_textile_element_not_opening_markup(_parser, [{:quicktxt, lit} | r_tokens]) do + {:ok, [{:text, escape(<>)}], r_tokens} + end + + defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens]) do + case repeat(&bq_cite_text/2, parser, r_tokens) do + {:ok, tree, [{:bq_cite_open, open} | r2_tokens]} -> + case repeat(&block_textile_element/2, parser, r2_tokens) do + {:ok, tree2, [{:bq_close, _} | r3_tokens]} -> + cite = escape(flatten(tree)) + + {:ok, + [ + {:markup, "
"}, + tree2, + {:markup, "
"} + ], r3_tokens} + + {:ok, tree2, r3_tokens} -> + {:ok, + [ + {:text, escape(start)}, + {:text, escape(flatten(tree))}, + {:text, escape(open)}, + tree2 + ], r3_tokens} + end + + _ -> + {:ok, [{:text, escape(start)}], r_tokens} + end + end + + defp inline_textile_element_not_opening_markup(_parser, [{:bq_cite_open, tok} | r_tokens]) do + {:ok, [{:text, escape(tok)}], r_tokens} + end + + defp inline_textile_element_not_opening_markup(parser, tokens) do + [ + {:bq_open, :bq_close, "
", "
"}, + {:spoiler_open, :spoiler_close, "||", "||"}, + {:bracketed_b_open, :bracketed_b_close, "**", "**"}, + {:bracketed_i_open, :bracketed_i_close, "_", "_"}, + {:bracketed_strong_open, :bracketed_strong_close, "**", "**"}, + {:bracketed_em_open, :bracketed_em_close, "*", "*"}, + {:bracketed_code_open, :bracketed_code_close, "```", "```"}, + {:bracketed_ins_open, :bracketed_ins_close, "__", "__"}, + {:bracketed_sup_open, :bracketed_sup_close, "^", "^"}, + {:bracketed_del_open, :bracketed_del_close, "~~", "~~"}, + {:bracketed_sub_open, :bracketed_sub_close, "%", "%"} + ] + |> Enum.find_value(fn {open_token, close_token, open_tag, close_tag} -> + simple_recursive( + open_token, + close_token, + open_tag, + close_tag, + &block_textile_element/2, + parser, + tokens + ) + |> case do + {:ok, tree, r_tokens} -> + {:ok, tree, r_tokens} + + _ -> + nil + end + end) + |> Kernel.||({:error, "Expected block markup"}) + end + + # + # block_textile_element = + # double_newline | newline | inline_textile_element; + # + + defp block_textile_element(_parser, [{:double_newline, _} | r_tokens]) do + {:ok, [{:markup, "\n"}], r_tokens} + end + + defp block_textile_element(_parser, [{:newline, _} | r_tokens]) do + {:ok, [{:markup, "\n\n"}], r_tokens} + end + + defp block_textile_element(parser, tokens) do + inline_textile_element(parser, tokens) + end + + # + # textile = + # (block_textile_element | TOKEN)* eos; + # + + defp textile(parser, tokens) do + case block_textile_element(parser, tokens) do + {:ok, tree, r_tokens} -> + {:ok, tree, r_tokens} + + _ -> + case tokens do + [{_, string} | r_tokens] -> + {:ok, [{:text, escape(string)}], r_tokens} + + _ -> + {:error, "Expected textile"} + end + end + end +end diff --git a/lib/philomena_web/textile_renderer.ex b/lib/philomena_web/textile_renderer.ex index d2ef7671..ade73e97 100644 --- a/lib/philomena_web/textile_renderer.ex +++ b/lib/philomena_web/textile_renderer.ex @@ -101,11 +101,11 @@ defmodule PhilomenaWeb.TextileRenderer do |> safe_to_string() [image, suffix] when suffix in ["p", "t", "s"] -> - link(">>#{image.id}#{suffix}#{link_postfix(image)}", to: "/#{image.id}") + link(">>#{image.id}#{suffix}#{link_postfix(image)}", to: "/images/#{image.id}") |> safe_to_string() [image] -> - link(">>#{image.id}#{link_postfix(image)}", to: "/#{image.id}") + link(">>#{image.id}#{link_postfix(image)}", to: "/images/#{image.id}") |> safe_to_string() end end) diff --git a/native/furbooru_markdown/src/lib.rs b/native/furbooru_markdown/src/lib.rs index 7d834349..16587cba 100644 --- a/native/furbooru_markdown/src/lib.rs +++ b/native/furbooru_markdown/src/lib.rs @@ -84,7 +84,6 @@ fn to_html_unsafe(input: String) -> String { options.extension.subscript = true; options.extension.spoiler = true; options.extension.strikethrough = true; - options.extension.front_matter_delimiter = Some("---".to_owned()); // options.extension.furbooru = true; options.parse.smart = true; options.render.hardbreaks = true; From cab0fb1b18fb15d3d70a705d478624c0013f52cc Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 17:53:31 +0200 Subject: [PATCH 06/25] underline + escaping md --- lib/philomena/markdown.ex | 6 ++++-- native/furbooru_markdown/Cargo.lock | 4 ++-- native/furbooru_markdown/Cargo.toml | 10 +++++----- native/furbooru_markdown/src/lib.rs | 2 ++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index fd8f8563..d25cac8d 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -1,7 +1,7 @@ defmodule Philomena.Markdown do use Rustler, otp_app: :philomena - @markdown_chars = ~r/[\*_\[\]\(\)\^`\%\\~<>#\|]/ + @markdown_chars ~r/[\*_\[\]\(\)\^`\%\\~<>#\|]/ # When your NIF is loaded, it will override this function. def to_html(_text), do: :erlang.nif_error(:nif_not_loaded) @@ -9,6 +9,8 @@ defmodule Philomena.Markdown do def escape_markdown(text) do @markdown_chars - |> Regex.replace(text, "\\\\0") + |> Regex.replace(text, fn m -> + "\\#{m}" + end) end end diff --git a/native/furbooru_markdown/Cargo.lock b/native/furbooru_markdown/Cargo.lock index bf632bd9..10243927 100644 --- a/native/furbooru_markdown/Cargo.lock +++ b/native/furbooru_markdown/Cargo.lock @@ -137,7 +137,7 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/furbooru/comrak#4adb01d736b03a25c81451d62148da37256e99c7" +source = "git+https://github.com/furbooru/comrak#b69625ef55b855675d9597c9dd5fe2dda50cc0c6" dependencies = [ "clap", "entities", @@ -223,7 +223,7 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "furbooru_markdown" -version = "0.1.1" +version = "0.2.0" dependencies = [ "comrak", "jemallocator", diff --git a/native/furbooru_markdown/Cargo.toml b/native/furbooru_markdown/Cargo.toml index 7c21e352..66ba6392 100644 --- a/native/furbooru_markdown/Cargo.toml +++ b/native/furbooru_markdown/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "furbooru_markdown" -version = "0.1.1" -authors = [] +version = "0.2.0" +authors = ["Xe ", "Luna ", "Liam White "] edition = "2018" [lib] @@ -11,11 +11,11 @@ crate-type = ["dylib"] [dependencies] comrak = { git = "https://github.com/furbooru/comrak" } -lazy_static = "1.0" -regex = "1" +lazy_static = "1.4" +regex = "1.5" log = "0" pretty_env_logger = "0" -rustler = "0.22.0" +rustler = "0.22" jemallocator = "0.3.2" [profile.release] diff --git a/native/furbooru_markdown/src/lib.rs b/native/furbooru_markdown/src/lib.rs index 16587cba..466439af 100644 --- a/native/furbooru_markdown/src/lib.rs +++ b/native/furbooru_markdown/src/lib.rs @@ -41,6 +41,7 @@ fn to_html(input: String) -> String { options.extension.subscript = true; options.extension.spoiler = true; options.extension.strikethrough = true; + options.extension.underline = true; // options.extension.furbooru = true; options.parse.smart = true; options.render.hardbreaks = true; @@ -84,6 +85,7 @@ fn to_html_unsafe(input: String) -> String { options.extension.subscript = true; options.extension.spoiler = true; options.extension.strikethrough = true; + options.extension.underline = true; // options.extension.furbooru = true; options.parse.smart = true; options.render.hardbreaks = true; From cb0ac0a9f512369249eacabff5f3cc3bf3c470c6 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 18:59:44 +0200 Subject: [PATCH 07/25] finalize textile-md converter --- lib/philomena/textile/parser_markdown.ex | 100 ++++++++++++++---- .../textile_markdown_renderer.ex | 22 ++++ 2 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 lib/philomena_web/textile_markdown_renderer.ex diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 0fb47133..20bdd371 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -1,3 +1,16 @@ +# LUNA PRESENTS THEE +# +# DA ULTIMATE, BESTEST, MOST SECURE AND DEFINITELY NOT BUGGY +# TEXTILE TO MARKDOWN CONVERTER PARSER LIBRARY THING!!!!! +# +# IT'S SO AWESOME I HAVE TO DESCRIBE IT IN ALL CAPS +# +# BY LOOKING AT THIS SOURCE CODE YOU AGREE THAT I MAY NOT BE HELD +# RESPONSIBLE FOR YOU DEVELOPING EYE CANCER +# +# YOU'VE BEEN WARNED +# +# COPYRIGHT (C) (R) (TM) LUNA (C) (R) (TM) 2021-206969696969 defmodule Philomena.Textile.ParserMarkdown do alias Philomena.Textile.Lexer alias Philomena.Markdown @@ -55,6 +68,17 @@ defmodule Philomena.Textile.ParserMarkdown do # Helper corresponding to Kleene star (*) operator # Match a specificed rule zero or more times + defp repeat(rule, parser, tokens, bq) when bq == true do + case rule.(parser, tokens, true) do + {:ok, tree, r_tokens} -> + {:ok, tree2, r2_tokens} = repeat(rule, parser, r_tokens, true) + {:ok, [tree, tree2], r2_tokens} + + _ -> + {:ok, [], tokens} + end + end + defp repeat(rule, parser, tokens) do case rule.(parser, tokens) do {:ok, tree, r_tokens} -> @@ -70,6 +94,18 @@ defmodule Philomena.Textile.ParserMarkdown do # # open_token callback* close_token # + defp simple_recursive(:bq_open, :bq_close, open_tag, close_tag, callback, parser, [ + {:bq_open, open} | r_tokens + ]) do + case repeat(callback, parser, r_tokens, true) do + {:ok, tree, [{:bq_close, _} | r2_tokens]} -> + {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} + + {:ok, tree, r2_tokens} -> + {:ok, [{:text, escape(open)}, tree], r2_tokens} + end + end + defp simple_recursive(open_token, close_token, open_tag, close_tag, callback, parser, [ {open_token, open} | r_tokens ]) do @@ -350,7 +386,13 @@ defmodule Philomena.Textile.ParserMarkdown do defp inline_textile_element_not_opening_markup(_parser, [{:char, lit} | r_tokens]) do {binary, r2_tokens} = assemble_binary(:char, <>, r_tokens) - {:ok, [{:text, escape(binary)}], r2_tokens} + # A bit of an ugly hack to force the parser to output >> instead of >> + case binary do + ">>" -> + {:ok, [{:text, binary}], r2_tokens} + _ -> + {:ok, [{:text, escape(binary)}], r2_tokens} + end end defp inline_textile_element_not_opening_markup(_parser, [ @@ -379,17 +421,15 @@ defmodule Philomena.Textile.ParserMarkdown do defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens]) do case repeat(&bq_cite_text/2, parser, r_tokens) do {:ok, tree, [{:bq_cite_open, open} | r2_tokens]} -> - case repeat(&block_textile_element/2, parser, r2_tokens) do + case repeat(&block_textile_element/3, parser, r2_tokens, true) do {:ok, tree2, [{:bq_close, _} | r3_tokens]} -> cite = escape(flatten(tree)) {:ok, [ - {:markup, "
"}, + {:markup, "\n> "}, tree2, - {:markup, "
"} + {:markup, "\n"} ], r3_tokens} {:ok, tree2, r3_tokens} -> @@ -413,7 +453,7 @@ defmodule Philomena.Textile.ParserMarkdown do defp inline_textile_element_not_opening_markup(parser, tokens) do [ - {:bq_open, :bq_close, "
", "
"}, + {:bq_open, :bq_close, "\n> ", "\n"}, {:spoiler_open, :spoiler_close, "||", "||"}, {:bracketed_b_open, :bracketed_b_close, "**", "**"}, {:bracketed_i_open, :bracketed_i_close, "_", "_"}, @@ -426,15 +466,27 @@ defmodule Philomena.Textile.ParserMarkdown do {:bracketed_sub_open, :bracketed_sub_close, "%", "%"} ] |> Enum.find_value(fn {open_token, close_token, open_tag, close_tag} -> - simple_recursive( - open_token, - close_token, - open_tag, - close_tag, - &block_textile_element/2, - parser, - tokens - ) + if open_token == :bq_open do + simple_recursive( + open_token, + close_token, + open_tag, + close_tag, + &block_textile_element/3, + parser, + tokens + ) + else + simple_recursive( + open_token, + close_token, + open_tag, + close_tag, + &block_textile_element/2, + parser, + tokens + ) + end |> case do {:ok, tree, r_tokens} -> {:ok, tree, r_tokens} @@ -451,12 +503,24 @@ defmodule Philomena.Textile.ParserMarkdown do # double_newline | newline | inline_textile_element; # + defp block_textile_element(_parser, [{:double_newline, _} | r_tokens], bq) when bq == true do + {:ok, [{:markup, "\n> \n> "}], r_tokens} + end + + defp block_textile_element(_parser, [{:newline, _} | r_tokens], bq) when bq == true do + {:ok, [{:markup, "\n> "}], r_tokens} + end + + defp block_textile_element(parser, tokens, _bq) do + inline_textile_element(parser, tokens) + end + defp block_textile_element(_parser, [{:double_newline, _} | r_tokens]) do - {:ok, [{:markup, "\n"}], r_tokens} + {:ok, [{:markup, "\n\n"}], r_tokens} end defp block_textile_element(_parser, [{:newline, _} | r_tokens]) do - {:ok, [{:markup, "\n\n"}], r_tokens} + {:ok, [{:markup, "\n"}], r_tokens} end defp block_textile_element(parser, tokens) do diff --git a/lib/philomena_web/textile_markdown_renderer.ex b/lib/philomena_web/textile_markdown_renderer.ex new file mode 100644 index 00000000..43d34787 --- /dev/null +++ b/lib/philomena_web/textile_markdown_renderer.ex @@ -0,0 +1,22 @@ +defmodule PhilomenaWeb.TextileMarkdownRenderer do + alias Philomena.Textile.ParserMarkdown + + def render_one(post, conn) do + hd(render_collection([post], conn)) + end + + def render_collection(posts, conn) do + opts = %{image_transform: &Camo.Image.image_url/1} + parsed = Enum.map(posts, &ParserMarkdown.parse(opts, &1.body)) + + parsed + |> Enum.map(fn tree -> + tree + |> Enum.map(fn + {_k, text} -> + text + end) + |> Enum.join() + end) + end +end From 14db174c26c4db53dc284932065b398c79c316c1 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 21:03:59 +0200 Subject: [PATCH 08/25] add *_md versions of all columns, textile/md glue --- lib/philomena/badges/badge.ex | 1 + lib/philomena/channels/channel.ex | 1 + lib/philomena/comments/comment.ex | 1 + lib/philomena/commissions/commission.ex | 4 + lib/philomena/commissions/item.ex | 2 + lib/philomena/conversations/message.ex | 1 + lib/philomena/dnp_entries/dnp_entry.ex | 3 + lib/philomena/filters/filter.ex | 1 + lib/philomena/galleries/gallery.ex | 1 + lib/philomena/images/image.ex | 2 + lib/philomena/mod_notes/mod_note.ex | 1 + lib/philomena/posts/post.ex | 1 + lib/philomena/reports/report.ex | 1 + lib/philomena/tags/tag.ex | 1 + lib/philomena/users/user.ex | 2 + .../controllers/admin/dnp_entry_controller.ex | 6 +- .../controllers/admin/mod_note_controller.ex | 4 +- .../controllers/admin/report_controller.ex | 6 +- .../controllers/comment_controller.ex | 4 +- .../controllers/conversation_controller.ex | 4 +- .../controllers/dnp_entry_controller.ex | 16 +- .../controllers/image/comment_controller.ex | 6 +- .../image/description_controller.ex | 4 +- .../controllers/image_controller.ex | 8 +- .../controllers/post/preview_controller.ex | 6 +- .../controllers/post_controller.ex | 4 +- .../profile/commission_controller.ex | 20 +-- .../controllers/profile/detail_controller.ex | 4 +- .../controllers/profile_controller.ex | 14 +- .../controllers/tag_controller.ex | 8 +- .../controllers/topic_controller.ex | 4 +- lib/philomena_web/image_loader.ex | 8 +- lib/philomena_web/markdown_renderer.ex | 7 + lib/philomena_web/text_renderer.ex | 18 +++ lib/philomena_web/textile_renderer.ex | 58 +++---- .../20210912171343_add_markdown_columns.exs | 145 ++++++++++++++++++ priv/repo/structure.sql | 58 +++++-- 37 files changed, 320 insertions(+), 115 deletions(-) create mode 100644 lib/philomena_web/markdown_renderer.ex create mode 100644 lib/philomena_web/text_renderer.ex create mode 100644 priv/repo/migrations/20210912171343_add_markdown_columns.exs diff --git a/lib/philomena/badges/badge.ex b/lib/philomena/badges/badge.ex index d7a232aa..88376d43 100644 --- a/lib/philomena/badges/badge.ex +++ b/lib/philomena/badges/badge.ex @@ -5,6 +5,7 @@ defmodule Philomena.Badges.Badge do schema "badges" do field :title, :string field :description, :string, default: "" + field :description_md, :string, default: "" field :image, :string field :disable_award, :boolean, default: false field :priority, :boolean, default: false diff --git a/lib/philomena/channels/channel.ex b/lib/philomena/channels/channel.ex index 1ace95a1..9c93b83c 100644 --- a/lib/philomena/channels/channel.ex +++ b/lib/philomena/channels/channel.ex @@ -14,6 +14,7 @@ defmodule Philomena.Channels.Channel do field :short_name, :string field :title, :string, default: "" field :description, :string + field :description_md, :string field :tags, :string field :viewers, :integer, default: 0 field :nsfw, :boolean, default: false diff --git a/lib/philomena/comments/comment.ex b/lib/philomena/comments/comment.ex index 180aaaaa..f7ee6f5d 100644 --- a/lib/philomena/comments/comment.ex +++ b/lib/philomena/comments/comment.ex @@ -11,6 +11,7 @@ defmodule Philomena.Comments.Comment do belongs_to :deleted_by, User field :body, :string + field :body_md, :string field :ip, EctoNetwork.INET field :fingerprint, :string field :user_agent, :string, default: "" diff --git a/lib/philomena/commissions/commission.ex b/lib/philomena/commissions/commission.ex index 18cb7b7b..c27fdbb2 100644 --- a/lib/philomena/commissions/commission.ex +++ b/lib/philomena/commissions/commission.ex @@ -17,6 +17,10 @@ defmodule Philomena.Commissions.Commission do field :contact, :string field :will_create, :string field :will_not_create, :string + field :information_md, :string + field :contact_md, :string + field :will_create_md, :string + field :will_not_create_md, :string field :commission_items_count, :integer, default: 0 timestamps(inserted_at: :created_at, type: :utc_datetime) diff --git a/lib/philomena/commissions/item.ex b/lib/philomena/commissions/item.ex index 59682f7a..f8b10c4d 100644 --- a/lib/philomena/commissions/item.ex +++ b/lib/philomena/commissions/item.ex @@ -11,8 +11,10 @@ defmodule Philomena.Commissions.Item do field :item_type, :string field :description, :string + field :description_md, :string field :base_price, :decimal field :add_ons, :string + field :add_ons_md, :string timestamps(inserted_at: :created_at, type: :utc_datetime) end diff --git a/lib/philomena/conversations/message.ex b/lib/philomena/conversations/message.ex index f1dc1129..365ad78b 100644 --- a/lib/philomena/conversations/message.ex +++ b/lib/philomena/conversations/message.ex @@ -10,6 +10,7 @@ defmodule Philomena.Conversations.Message do belongs_to :from, User field :body, :string + field :body_md, :string timestamps(inserted_at: :created_at, type: :utc_datetime) end diff --git a/lib/philomena/dnp_entries/dnp_entry.ex b/lib/philomena/dnp_entries/dnp_entry.ex index e59ab21e..14b5e90f 100644 --- a/lib/philomena/dnp_entries/dnp_entry.ex +++ b/lib/philomena/dnp_entries/dnp_entry.ex @@ -17,6 +17,9 @@ defmodule Philomena.DnpEntries.DnpEntry do field :hide_reason, :boolean, default: false field :instructions, :string, default: "" field :feedback, :string, default: "" + field :conditions_md, :string, default: "" + field :reason_md, :string, default: "" + field :instructions_md, :string, default: "" timestamps(inserted_at: :created_at, type: :utc_datetime) end diff --git a/lib/philomena/filters/filter.ex b/lib/philomena/filters/filter.ex index ea862b2b..e00dbcba 100644 --- a/lib/philomena/filters/filter.ex +++ b/lib/philomena/filters/filter.ex @@ -12,6 +12,7 @@ defmodule Philomena.Filters.Filter do field :name, :string field :description, :string, default: "" + field :description_md, :string, default: "" field :system, :boolean field :public, :boolean field :hidden_complex_str, :string diff --git a/lib/philomena/galleries/gallery.ex b/lib/philomena/galleries/gallery.ex index 9be09592..8225bf19 100644 --- a/lib/philomena/galleries/gallery.ex +++ b/lib/philomena/galleries/gallery.ex @@ -17,6 +17,7 @@ defmodule Philomena.Galleries.Gallery do field :title, :string field :spoiler_warning, :string, default: "" field :description, :string, default: "" + field :description_md, :string, default: "" field :image_count, :integer field :order_position_asc, :boolean diff --git a/lib/philomena/images/image.ex b/lib/philomena/images/image.ex index bbb7cba8..a302d70a 100644 --- a/lib/philomena/images/image.ex +++ b/lib/philomena/images/image.ex @@ -64,6 +64,7 @@ defmodule Philomena.Images.Image do field :votes_count, :integer, default: 0 field :source_url, :string field :description, :string, default: "" + field :description_md, :string, default: "" field :image_sha512_hash, :string field :image_orig_sha512_hash, :string field :deletion_reason, :string @@ -80,6 +81,7 @@ defmodule Philomena.Images.Image do field :destroyed_content, :boolean field :hidden_image_key, :string field :scratchpad, :string + field :scratchpad_md, :string field :hides_count, :integer, default: 0 # todo: can probably remove these now diff --git a/lib/philomena/mod_notes/mod_note.ex b/lib/philomena/mod_notes/mod_note.ex index 8344b7ec..acc3555b 100644 --- a/lib/philomena/mod_notes/mod_note.ex +++ b/lib/philomena/mod_notes/mod_note.ex @@ -12,6 +12,7 @@ defmodule Philomena.ModNotes.ModNote do field :notable_type, :string field :body, :string + field :body_md, :string field :notable, :any, virtual: true diff --git a/lib/philomena/posts/post.ex b/lib/philomena/posts/post.ex index b3c3c42f..1e2c2b96 100644 --- a/lib/philomena/posts/post.ex +++ b/lib/philomena/posts/post.ex @@ -11,6 +11,7 @@ defmodule Philomena.Posts.Post do belongs_to :deleted_by, User field :body, :string + field :body_md, :string field :edit_reason, :string field :ip, EctoNetwork.INET field :fingerprint, :string diff --git a/lib/philomena/reports/report.ex b/lib/philomena/reports/report.ex index 8f8de00b..5fffe5ec 100644 --- a/lib/philomena/reports/report.ex +++ b/lib/philomena/reports/report.ex @@ -13,6 +13,7 @@ defmodule Philomena.Reports.Report do field :user_agent, :string, default: "" field :referrer, :string, default: "" field :reason, :string + field :reason_md, :string field :state, :string, default: "open" field :open, :boolean, default: true diff --git a/lib/philomena/tags/tag.ex b/lib/philomena/tags/tag.ex index 2713a392..fe5e9491 100644 --- a/lib/philomena/tags/tag.ex +++ b/lib/philomena/tags/tag.ex @@ -77,6 +77,7 @@ defmodule Philomena.Tags.Tag do field :category, :string field :images_count, :integer, default: 0 field :description, :string + field :description_md, :string field :short_description, :string field :namespace, :string field :name_in_namespace, :string diff --git a/lib/philomena/users/user.ex b/lib/philomena/users/user.ex index dc41404d..df00d533 100644 --- a/lib/philomena/users/user.ex +++ b/lib/philomena/users/user.ex @@ -65,6 +65,7 @@ defmodule Philomena.Users.User do field :slug, :string field :role, :string, default: "user" field :description, :string + field :description_md, :string field :avatar, :string # Settings @@ -115,6 +116,7 @@ defmodule Philomena.Users.User do field :last_renamed_at, :utc_datetime field :deleted_at, :utc_datetime field :scratchpad, :string + field :scratchpad_md, :string field :secondary_role, :string field :hide_default_role, :boolean, default: false field :senior_staff, :boolean, default: false diff --git a/lib/philomena_web/controllers/admin/dnp_entry_controller.ex b/lib/philomena_web/controllers/admin/dnp_entry_controller.ex index 3e53d7ab..b389f49e 100644 --- a/lib/philomena_web/controllers/admin/dnp_entry_controller.ex +++ b/lib/philomena_web/controllers/admin/dnp_entry_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.Admin.DnpEntryController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.DnpEntries.DnpEntry alias Philomena.Repo import Ecto.Query @@ -44,8 +44,8 @@ defmodule PhilomenaWeb.Admin.DnpEntryController do bodies = dnp_entries - |> Enum.map(&%{body: &1.conditions}) - |> TextileRenderer.render_collection(conn) + |> Enum.map(&%{body: &1.conditions, body_md: &1.conditions_md}) + |> TextRenderer.render_collection(conn) dnp_entries = %{dnp_entries | entries: Enum.zip(bodies, dnp_entries.entries)} diff --git a/lib/philomena_web/controllers/admin/mod_note_controller.ex b/lib/philomena_web/controllers/admin/mod_note_controller.ex index 53d91eb1..275bd7e0 100644 --- a/lib/philomena_web/controllers/admin/mod_note_controller.ex +++ b/lib/philomena_web/controllers/admin/mod_note_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.Admin.ModNoteController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.ModNotes.ModNote alias Philomena.Polymorphic alias Philomena.ModNotes @@ -29,7 +29,7 @@ defmodule PhilomenaWeb.Admin.ModNoteController do |> order_by(desc: :id) |> Repo.paginate(conn.assigns.scrivener) - bodies = TextileRenderer.render_collection(mod_notes, conn) + bodies = TextRenderer.render_collection(mod_notes, conn) preloaded = Polymorphic.load_polymorphic(mod_notes, notable: [notable_id: :notable_type]) mod_notes = %{mod_notes | entries: Enum.zip(bodies, preloaded)} diff --git a/lib/philomena_web/controllers/admin/report_controller.ex b/lib/philomena_web/controllers/admin/report_controller.ex index 40fbf191..1a40356f 100644 --- a/lib/philomena_web/controllers/admin/report_controller.ex +++ b/lib/philomena_web/controllers/admin/report_controller.ex @@ -2,7 +2,7 @@ defmodule PhilomenaWeb.Admin.ReportController do use PhilomenaWeb, :controller alias Philomena.Elasticsearch - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Reports.Report alias Philomena.Reports.Query alias Philomena.Polymorphic @@ -73,7 +73,7 @@ defmodule PhilomenaWeb.Admin.ReportController do reportable: [reportable_id: :reportable_type] ) - body = TextileRenderer.render_one(%{body: report.reason}, conn) + body = TextRenderer.render_one(%{body: report.reason, body_md: report.reason_md}, conn) render(conn, "show.html", title: "Showing Report", report: report, body: body) end @@ -125,7 +125,7 @@ defmodule PhilomenaWeb.Admin.ReportController do mod_notes = mod_notes - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) |> Enum.zip(mod_notes) assign(conn, :mod_notes, mod_notes) diff --git a/lib/philomena_web/controllers/comment_controller.ex b/lib/philomena_web/controllers/comment_controller.ex index f494fe08..4f2f4ef0 100644 --- a/lib/philomena_web/controllers/comment_controller.ex +++ b/lib/philomena_web/controllers/comment_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.CommentController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Elasticsearch alias Philomena.{Comments.Query, Comments.Comment} import Ecto.Query @@ -39,7 +39,7 @@ defmodule PhilomenaWeb.CommentController do preload(Comment, [:deleted_by, image: [tags: :aliases], user: [awards: :badge]]) ) - rendered = TextileRenderer.render_collection(comments.entries, conn) + rendered = TextRenderer.render_collection(comments.entries, conn) comments = %{comments | entries: Enum.zip(rendered, comments.entries)} diff --git a/lib/philomena_web/controllers/conversation_controller.ex b/lib/philomena_web/controllers/conversation_controller.ex index 1292c441..9a843df4 100644 --- a/lib/philomena_web/controllers/conversation_controller.ex +++ b/lib/philomena_web/controllers/conversation_controller.ex @@ -3,7 +3,7 @@ defmodule PhilomenaWeb.ConversationController do alias PhilomenaWeb.NotificationCountPlug alias Philomena.{Conversations, Conversations.Conversation, Conversations.Message} - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Repo import Ecto.Query @@ -72,7 +72,7 @@ defmodule PhilomenaWeb.ConversationController do rendered = messages.entries - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) messages = %{messages | entries: Enum.zip(messages.entries, rendered)} diff --git a/lib/philomena_web/controllers/dnp_entry_controller.ex b/lib/philomena_web/controllers/dnp_entry_controller.ex index f4dcfca5..38f7127a 100644 --- a/lib/philomena_web/controllers/dnp_entry_controller.ex +++ b/lib/philomena_web/controllers/dnp_entry_controller.ex @@ -2,7 +2,7 @@ defmodule PhilomenaWeb.DnpEntryController do use PhilomenaWeb, :controller alias Philomena.DnpEntries.DnpEntry - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.DnpEntries alias Philomena.Tags.Tag alias Philomena.ModNotes.ModNote @@ -43,8 +43,8 @@ defmodule PhilomenaWeb.DnpEntryController do bodies = dnp_entries - |> Enum.map(&%{body: &1.conditions || "-"}) - |> TextileRenderer.render_collection(conn) + |> Enum.map(&%{body_md: &1.conditions_md, body: &1.conditions || "-"}) + |> TextRenderer.render_collection(conn) dnp_entries = %{dnp_entries | entries: Enum.zip(bodies, dnp_entries.entries)} @@ -61,11 +61,11 @@ defmodule PhilomenaWeb.DnpEntryController do dnp_entry = conn.assigns.dnp_entry [conditions, reason, instructions] = - TextileRenderer.render_collection( + TextRenderer.render_collection( [ - %{body: dnp_entry.conditions || "-"}, - %{body: dnp_entry.reason || "-"}, - %{body: dnp_entry.instructions || "-"} + %{body_md: dnp_entry.conditions_md, body: dnp_entry.conditions || "-"}, + %{body_md: dnp_entry.reason_md, body: dnp_entry.reason || "-"}, + %{body_md: dnp_entry.instructions_md, body: dnp_entry.instructions || "-"} ], conn ) @@ -164,7 +164,7 @@ defmodule PhilomenaWeb.DnpEntryController do mod_notes = mod_notes - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) |> Enum.zip(mod_notes) assign(conn, :mod_notes, mod_notes) diff --git a/lib/philomena_web/controllers/image/comment_controller.ex b/lib/philomena_web/controllers/image/comment_controller.ex index 3e75cc76..ebb605c3 100644 --- a/lib/philomena_web/controllers/image/comment_controller.ex +++ b/lib/philomena_web/controllers/image/comment_controller.ex @@ -2,7 +2,7 @@ defmodule PhilomenaWeb.Image.CommentController do use PhilomenaWeb, :controller alias PhilomenaWeb.CommentLoader - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.{Images.Image, Comments.Comment} alias Philomena.UserStatistics alias Philomena.Comments @@ -48,7 +48,7 @@ defmodule PhilomenaWeb.Image.CommentController do def index(conn, _params) do comments = CommentLoader.load_comments(conn, conn.assigns.image) - rendered = TextileRenderer.render_collection(comments.entries, conn) + rendered = TextRenderer.render_collection(comments.entries, conn) comments = %{comments | entries: Enum.zip(comments.entries, rendered)} @@ -56,7 +56,7 @@ defmodule PhilomenaWeb.Image.CommentController do end def show(conn, _params) do - rendered = TextileRenderer.render_one(conn.assigns.comment, conn) + rendered = TextRenderer.render_one(conn.assigns.comment, conn) render(conn, "show.html", layout: false, diff --git a/lib/philomena_web/controllers/image/description_controller.ex b/lib/philomena_web/controllers/image/description_controller.ex index f09f12fe..7e508567 100644 --- a/lib/philomena_web/controllers/image/description_controller.ex +++ b/lib/philomena_web/controllers/image/description_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.Image.DescriptionController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Images.Image alias Philomena.Images @@ -34,7 +34,7 @@ defmodule PhilomenaWeb.Image.DescriptionController do Images.reindex_image(image) - body = TextileRenderer.render_one(%{body: image.description}, conn) + body = TextRenderer.render_one(%{body: image.description, body_md: image.description_md}, conn) conn |> put_view(PhilomenaWeb.ImageView) diff --git a/lib/philomena_web/controllers/image_controller.ex b/lib/philomena_web/controllers/image_controller.ex index 31f94104..10c9dc47 100644 --- a/lib/philomena_web/controllers/image_controller.ex +++ b/lib/philomena_web/controllers/image_controller.ex @@ -4,7 +4,7 @@ defmodule PhilomenaWeb.ImageController do alias PhilomenaWeb.ImageLoader alias PhilomenaWeb.CommentLoader alias PhilomenaWeb.NotificationCountPlug - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.{ Images, @@ -63,13 +63,13 @@ defmodule PhilomenaWeb.ImageController do comments = CommentLoader.load_comments(conn, image) - rendered = TextileRenderer.render_collection(comments.entries, conn) + rendered = TextRenderer.render_collection(comments.entries, conn) comments = %{comments | entries: Enum.zip(comments.entries, rendered)} description = - %{body: image.description} - |> TextileRenderer.render_one(conn) + %{body: image.description, body_md: image.description_md} + |> TextRenderer.render_one(conn) interactions = Interactions.user_interactions([image], conn.assigns.current_user) diff --git a/lib/philomena_web/controllers/post/preview_controller.ex b/lib/philomena_web/controllers/post/preview_controller.ex index 00416436..b5a34e75 100644 --- a/lib/philomena_web/controllers/post/preview_controller.ex +++ b/lib/philomena_web/controllers/post/preview_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.Post.PreviewController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Posts.Post alias Philomena.Repo @@ -10,8 +10,8 @@ defmodule PhilomenaWeb.Post.PreviewController do body = to_string(params["body"]) anonymous = params["anonymous"] == true - post = %Post{user: user, body: body, anonymous: anonymous} - rendered = TextileRenderer.render_one(post, conn) + post = %Post{user: user, body_md: body, anonymous: anonymous} + rendered = TextRenderer.render_one(post, conn) render(conn, "create.html", layout: false, post: post, body: rendered) end diff --git a/lib/philomena_web/controllers/post_controller.ex b/lib/philomena_web/controllers/post_controller.ex index 0685d1ab..e849b5ba 100644 --- a/lib/philomena_web/controllers/post_controller.ex +++ b/lib/philomena_web/controllers/post_controller.ex @@ -1,7 +1,7 @@ defmodule PhilomenaWeb.PostController do use PhilomenaWeb, :controller - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Elasticsearch alias Philomena.{Posts.Query, Posts.Post} import Ecto.Query @@ -36,7 +36,7 @@ defmodule PhilomenaWeb.PostController do preload(Post, [:deleted_by, topic: :forum, user: [awards: :badge]]) ) - rendered = TextileRenderer.render_collection(posts.entries, conn) + rendered = TextRenderer.render_collection(posts.entries, conn) posts = %{posts | entries: Enum.zip(rendered, posts.entries)} diff --git a/lib/philomena_web/controllers/profile/commission_controller.ex b/lib/philomena_web/controllers/profile/commission_controller.ex index e677fc4f..0a9134d2 100644 --- a/lib/philomena_web/controllers/profile/commission_controller.ex +++ b/lib/philomena_web/controllers/profile/commission_controller.ex @@ -3,7 +3,7 @@ defmodule PhilomenaWeb.Profile.CommissionController do alias Philomena.Commissions.Commission alias Philomena.Commissions - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Users.User plug PhilomenaWeb.FilterBannedUsersPlug when action in [:new, :create, :edit, :update, :delete] @@ -36,21 +36,21 @@ defmodule PhilomenaWeb.Profile.CommissionController do item_descriptions = items - |> Enum.map(&%{body: &1.description}) - |> TextileRenderer.render_collection(conn) + |> Enum.map(&%{body: &1.description, body_md: &1.description_md}) + |> TextRenderer.render_collection(conn) item_add_ons = items - |> Enum.map(&%{body: &1.add_ons}) - |> TextileRenderer.render_collection(conn) + |> Enum.map(&%{body: &1.add_ons, body_md: &1.add_ons_md}) + |> TextRenderer.render_collection(conn) [information, contact, will_create, will_not_create] = - TextileRenderer.render_collection( + TextRenderer.render_collection( [ - %{body: commission.information || ""}, - %{body: commission.contact || ""}, - %{body: commission.will_create || ""}, - %{body: commission.will_not_create || ""} + %{body_md: commission.information_md, body: commission.information || ""}, + %{body_md: commission.contact_md, body: commission.contact || ""}, + %{body_md: commission.will_create_md, body: commission.will_create || ""}, + %{body_md: commission.will_not_create_md, body: commission.will_not_create || ""} ], conn ) diff --git a/lib/philomena_web/controllers/profile/detail_controller.ex b/lib/philomena_web/controllers/profile/detail_controller.ex index ced52900..17e88b0f 100644 --- a/lib/philomena_web/controllers/profile/detail_controller.ex +++ b/lib/philomena_web/controllers/profile/detail_controller.ex @@ -3,7 +3,7 @@ defmodule PhilomenaWeb.Profile.DetailController do alias Philomena.UserNameChanges.UserNameChange alias Philomena.ModNotes.ModNote - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Polymorphic alias Philomena.Users.User alias Philomena.Repo @@ -30,7 +30,7 @@ defmodule PhilomenaWeb.Profile.DetailController do mod_notes = mod_notes - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) |> Enum.zip(mod_notes) name_changes = diff --git a/lib/philomena_web/controllers/profile_controller.ex b/lib/philomena_web/controllers/profile_controller.ex index 6388d25b..b0cb6a2d 100644 --- a/lib/philomena_web/controllers/profile_controller.ex +++ b/lib/philomena_web/controllers/profile_controller.ex @@ -3,7 +3,7 @@ defmodule PhilomenaWeb.ProfileController do alias PhilomenaWeb.ImageLoader alias Philomena.Elasticsearch - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.UserStatistics.UserStatistic alias Philomena.Users.User alias Philomena.Bans @@ -131,12 +131,12 @@ defmodule PhilomenaWeb.ProfileController do recent_comments = recent_comments |> Enum.filter(&Canada.Can.can?(current_user, :show, &1.image)) - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) |> Enum.zip(recent_comments) - about_me = TextileRenderer.render_one(%{body: user.description || ""}, conn) + about_me = TextRenderer.render_one(%{body_md: user.description_md, body: user.description || ""}, conn) - scratchpad = TextileRenderer.render_one(%{body: user.scratchpad || ""}, conn) + scratchpad = TextRenderer.render_one(%{body_md: user.scratchpad_md, body: user.scratchpad || ""}, conn) commission_information = commission_info(user.commission, conn) @@ -214,8 +214,8 @@ defmodule PhilomenaWeb.ProfileController do defp map_fetch(nil, _field_name), do: nil defp map_fetch(map, field_name), do: Map.get(map, field_name) - defp commission_info(%{information: info}, conn) when info not in [nil, ""], - do: TextileRenderer.render_one(%{body: info}, conn) + defp commission_info(%{information: info, information_md: info_md}, conn) when info not in [nil, ""], + do: TextRenderer.render_one(%{body: info, body_md: info_md}, conn) defp commission_info(_commission, _conn), do: "" @@ -282,7 +282,7 @@ defmodule PhilomenaWeb.ProfileController do mod_notes = mod_notes - |> TextileRenderer.render_collection(conn) + |> TextRenderer.render_collection(conn) |> Enum.zip(mod_notes) assign(conn, :mod_notes, mod_notes) diff --git a/lib/philomena_web/controllers/tag_controller.ex b/lib/philomena_web/controllers/tag_controller.ex index 7f55bcf7..ee4c6f3b 100644 --- a/lib/philomena_web/controllers/tag_controller.ex +++ b/lib/philomena_web/controllers/tag_controller.ex @@ -5,7 +5,7 @@ defmodule PhilomenaWeb.TagController do alias Philomena.Elasticsearch alias Philomena.{Tags, Tags.Tag} alias Philomena.{Images, Images.Image} - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Interactions import Ecto.Query @@ -61,11 +61,11 @@ defmodule PhilomenaWeb.TagController do interactions = Interactions.user_interactions(images, user) - body = TextileRenderer.render_one(%{body: tag.description || ""}, conn) + body = TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) dnp_bodies = - TextileRenderer.render_collection( - Enum.map(tag.dnp_entries, &%{body: &1.conditions || ""}), + TextRenderer.render_collection( + Enum.map(tag.dnp_entries, &%{body_md: &1.conditions_md, body: &1.conditions || ""}), conn ) diff --git a/lib/philomena_web/controllers/topic_controller.ex b/lib/philomena_web/controllers/topic_controller.ex index e9eeea3d..536d9336 100644 --- a/lib/philomena_web/controllers/topic_controller.ex +++ b/lib/philomena_web/controllers/topic_controller.ex @@ -5,7 +5,7 @@ defmodule PhilomenaWeb.TopicController do alias Philomena.{Forums.Forum, Topics.Topic, Posts.Post, Polls.Poll, PollOptions.PollOption} alias Philomena.{Forums, Topics, Polls, Posts} alias Philomena.PollVotes - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Repo import Ecto.Query @@ -60,7 +60,7 @@ defmodule PhilomenaWeb.TopicController do |> preload([:deleted_by, :topic, topic: :forum, user: [awards: :badge]]) |> Repo.all() - rendered = TextileRenderer.render_collection(posts, conn) + rendered = TextRenderer.render_collection(posts, conn) posts = Enum.zip(posts, rendered) diff --git a/lib/philomena_web/image_loader.ex b/lib/philomena_web/image_loader.ex index 57035234..914404e6 100644 --- a/lib/philomena_web/image_loader.ex +++ b/lib/philomena_web/image_loader.ex @@ -2,7 +2,7 @@ defmodule PhilomenaWeb.ImageLoader do alias PhilomenaWeb.ImageSorter alias Philomena.Elasticsearch alias Philomena.Images.{Image, Query} - alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.TextRenderer alias Philomena.Tags.Tag alias Philomena.Repo import Ecto.Query @@ -132,14 +132,14 @@ defmodule PhilomenaWeb.ImageLoader do defp render_bodies([tag], conn) do dnp_bodies = - TextileRenderer.render_collection( - Enum.map(tag.dnp_entries, &%{body: &1.conditions || ""}), + TextRenderer.render_collection( + Enum.map(tag.dnp_entries, &%{body_md: &1.conditions_md, body: &1.conditions || ""}), conn ) dnp_entries = Enum.zip(dnp_bodies, tag.dnp_entries) - description = TextileRenderer.render_one(%{body: tag.description || ""}, conn) + description = TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) [{tag, description, dnp_entries}] end diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex new file mode 100644 index 00000000..72e7376a --- /dev/null +++ b/lib/philomena_web/markdown_renderer.ex @@ -0,0 +1,7 @@ +defmodule PhilomenaWeb.MarkdownRenderer do + alias Philomena.Markdown + + def render(text, _conn) do + Markdown.to_html(text) + end +end diff --git a/lib/philomena_web/text_renderer.ex b/lib/philomena_web/text_renderer.ex new file mode 100644 index 00000000..96152a50 --- /dev/null +++ b/lib/philomena_web/text_renderer.ex @@ -0,0 +1,18 @@ +defmodule PhilomenaWeb.TextRenderer do + alias PhilomenaWeb.TextileRenderer + alias PhilomenaWeb.MarkdownRenderer + + def render_one(item, conn) do + hd(render_collection([item], conn)) + end + + def render_collection(items, conn) do + Enum.map(items, fn item -> + if Map.has_key?(item, :body_md) && item.body_md != nil && item.body_md != "" do + MarkdownRenderer.render(item.body_md, conn) + else + TextileRenderer.render(item.body, conn) + end + end) + end +end diff --git a/lib/philomena_web/textile_renderer.ex b/lib/philomena_web/textile_renderer.ex index ade73e97..50870dfd 100644 --- a/lib/philomena_web/textile_renderer.ex +++ b/lib/philomena_web/textile_renderer.ex @@ -9,44 +9,32 @@ defmodule PhilomenaWeb.TextileRenderer do # Kill bogus compile time dependency on ImageView @image_view Module.concat(["PhilomenaWeb.ImageView"]) - def render_one(post, conn) do - hd(render_collection([post], conn)) - end + def render(text, conn) do + opts = %{image_transform: &Camo.Image.image_url/1} + parsed = Parser.parse(opts, text) + + images = + parsed + |> Enum.flat_map(fn + {:text, text} -> + [text] - def render_collection(posts, conn) do - # opts = %{image_transform: &Camo.Image.image_url/1} - # parsed = Enum.map(posts, &Parser.parse(opts, &1.body)) + _ -> + [] + end) + |> find_images - Enum.map(posts, &Philomena.Markdown.to_html(&1.body)) + parsed + |> Enum.map(fn + {:text, text} -> + text + |> replacement_entities() + |> replacement_images(conn, images) - # images = - # parsed - # |> Enum.flat_map(fn tree -> - # tree - # |> Enum.flat_map(fn - # {:text, text} -> - # [text] - - # _ -> - # [] - # end) - # end) - # |> find_images - - # parsed - # |> Enum.map(fn tree -> - # tree - # |> Enum.map(fn - # {:text, text} -> - # text - # |> replacement_entities() - # |> replacement_images(conn, images) - - # {_k, markup} -> - # markup - # end) - # |> Enum.join() - # end) + {_k, markup} -> + markup + end) + |> Enum.join() end defp replacement_entities(t) do diff --git a/priv/repo/migrations/20210912171343_add_markdown_columns.exs b/priv/repo/migrations/20210912171343_add_markdown_columns.exs new file mode 100644 index 00000000..21267871 --- /dev/null +++ b/priv/repo/migrations/20210912171343_add_markdown_columns.exs @@ -0,0 +1,145 @@ +defmodule Philomena.Repo.Migrations.AddMarkdownColumns do + use Ecto.Migration + + def up do + alter table("comments") do + add :body_md, :varchar, default: nil + end + + alter table("messages") do + add :body_md, :varchar, default: nil + end + + alter table("mod_notes") do + add :body_md, :varchar, default: nil + end + + alter table("posts") do + add :body_md, :varchar, default: nil + end + + alter table("badges") do + add :description_md, :varchar, default: nil + end + + alter table("channels") do + add :description_md, :varchar, default: nil + end + + alter table("commission_items") do + add :description_md, :varchar, default: nil + add :add_ons_md, :varchar, default: nil + end + + alter table("filters") do + add :description_md, :varchar, default: nil + end + + alter table("galleries") do + add :description_md, :varchar, default: nil + end + + alter table("images") do + add :description_md, :varchar, default: nil + add :scratchpad_md, :varchar, default: nil + end + + alter table("tags") do + add :description_md, :varchar, default: nil + end + + alter table("users") do + add :description_md, :varchar, default: nil + add :scratchpad_md, :varchar, default: nil + end + + alter table("dnp_entries") do + add :conditions_md, :varchar, default: nil + add :reason_md, :varchar, default: nil + add :instructions_md, :varchar, default: nil + end + + alter table("commissions") do + add :information_md, :varchar, default: nil + add :contact_md, :varchar, default: nil + add :will_create_md, :varchar, default: nil + add :will_not_create_md, :varchar, default: nil + end + + alter table("reports") do + add :reason_md, :varchar, default: nil + end + end + + def down do + alter table("comments") do + remove :body_md + end + + alter table("messages") do + remove :body_md + end + + alter table("mod_notes") do + remove :body_md + end + + alter table("posts") do + remove :body_md + end + + alter table("badges") do + remove :description_md + end + + alter table("channels") do + remove :description_md + end + + alter table("commission_items") do + remove :description_md + remove :add_ons_md + end + + alter table("filters") do + remove :description_md + end + + alter table("galleries") do + remove :description_md + end + + alter table("images") do + remove :description_md + remove :scratchpad_md + end + + alter table("tags") do + remove :description_md + remove :short_description_md + remove :mod_notes_md + end + + alter table("users") do + remove :description_md + remove :scratchpad_md + end + + alter table("dnp_entries") do + remove :conditions_md + remove :reason_md + remove :instructions_md + end + + alter table("commissions") do + remove :information_md + remove :contact_md + remove :will_create_md + remove :will_not_create_md + end + + alter table("reports") do + remove :reason_md + end + end +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 3337ab7d..b53aabc2 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 13.2 --- Dumped by pg_dump version 13.2 +-- Dumped from database version 13.4 +-- Dumped by pg_dump version 13.4 SET statement_timeout = 0; SET lock_timeout = 0; @@ -165,7 +165,8 @@ CREATE TABLE public.badges ( created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, disable_award boolean DEFAULT false NOT NULL, - priority boolean DEFAULT false + priority boolean DEFAULT false, + description_md character varying ); @@ -227,7 +228,8 @@ CREATE TABLE public.channels ( total_viewer_minutes integer DEFAULT 0 NOT NULL, banner_image character varying, remote_stream_id integer, - thumbnail_url character varying DEFAULT ''::character varying + thumbnail_url character varying DEFAULT ''::character varying, + description_md character varying ); @@ -272,7 +274,8 @@ CREATE TABLE public.comments ( edited_at timestamp without time zone, deletion_reason character varying DEFAULT ''::character varying NOT NULL, destroyed_content boolean DEFAULT false, - name_at_post_time character varying + name_at_post_time character varying, + body_md character varying ); @@ -308,7 +311,9 @@ CREATE TABLE public.commission_items ( add_ons character varying, example_image_id integer, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + description_md character varying, + add_ons_md character varying ); @@ -347,7 +352,11 @@ CREATE TABLE public.commissions ( will_not_create character varying, commission_items_count integer DEFAULT 0 NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + information_md character varying, + contact_md character varying, + will_create_md character varying, + will_not_create_md character varying ); @@ -426,7 +435,10 @@ CREATE TABLE public.dnp_entries ( instructions character varying NOT NULL, feedback character varying NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + conditions_md character varying, + reason_md character varying, + instructions_md character varying ); @@ -539,7 +551,8 @@ CREATE TABLE public.filters ( user_count integer DEFAULT 0 NOT NULL, created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, - user_id integer + user_id integer, + description_md character varying ); @@ -666,7 +679,8 @@ CREATE TABLE public.galleries ( watcher_ids integer[] DEFAULT '{}'::integer[] NOT NULL, watcher_count integer DEFAULT 0 NOT NULL, image_count integer DEFAULT 0 NOT NULL, - order_position_asc boolean DEFAULT false NOT NULL + order_position_asc boolean DEFAULT false NOT NULL, + description_md character varying ); @@ -950,7 +964,9 @@ CREATE TABLE public.images ( hidden_image_key character varying, scratchpad character varying, hides_count integer DEFAULT 0 NOT NULL, - image_duration double precision + image_duration double precision, + description_md character varying, + scratchpad_md character varying ); @@ -983,7 +999,8 @@ CREATE TABLE public.messages ( created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, from_id integer NOT NULL, - conversation_id integer NOT NULL + conversation_id integer NOT NULL, + body_md character varying ); @@ -1018,7 +1035,8 @@ CREATE TABLE public.mod_notes ( body text NOT NULL, deleted boolean DEFAULT false NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + body_md character varying ); @@ -1201,7 +1219,8 @@ CREATE TABLE public.posts ( edited_at timestamp without time zone, deletion_reason character varying DEFAULT ''::character varying NOT NULL, destroyed_content boolean DEFAULT false NOT NULL, - name_at_post_time character varying + name_at_post_time character varying, + body_md character varying ); @@ -1242,7 +1261,8 @@ CREATE TABLE public.reports ( user_id integer, admin_id integer, reportable_id integer NOT NULL, - reportable_type character varying NOT NULL + reportable_type character varying NOT NULL, + reason_md character varying ); @@ -1549,7 +1569,8 @@ CREATE TABLE public.tags ( created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, category character varying, - mod_notes character varying + mod_notes character varying, + description_md character varying ); @@ -1988,7 +2009,9 @@ CREATE TABLE public.users ( last_renamed_at timestamp without time zone DEFAULT '1970-01-01 00:00:00'::timestamp without time zone NOT NULL, forced_filter_id bigint, confirmed_at timestamp(0) without time zone, - senior_staff boolean DEFAULT false + senior_staff boolean DEFAULT false, + description_md character varying, + scratchpad_md character varying ); @@ -4844,3 +4867,4 @@ INSERT INTO public."schema_migrations" (version) VALUES (20201124224116); INSERT INTO public."schema_migrations" (version) VALUES (20210121200815); INSERT INTO public."schema_migrations" (version) VALUES (20210301012137); INSERT INTO public."schema_migrations" (version) VALUES (20210427022351); +INSERT INTO public."schema_migrations" (version) VALUES (20210912171343); From 8a1ec4ce732889fc4816f1efa820597fb30636a2 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 12 Sep 2021 22:07:43 +0200 Subject: [PATCH 09/25] rename furbooru_markdown to philomena_markdown --- config/config.exs | 2 +- mix.exs | 2 +- mix.lock | 2 +- .../.cargo/config | 0 .../Cargo.lock | 29 +++++++++---------- .../Cargo.toml | 6 ++-- .../README.md | 6 ++-- .../src/lib.rs | 10 ++----- 8 files changed, 25 insertions(+), 32 deletions(-) rename native/{furbooru_markdown => philomena_markdown}/.cargo/config (100%) rename native/{furbooru_markdown => philomena_markdown}/Cargo.lock (99%) rename native/{furbooru_markdown => philomena_markdown}/Cargo.toml (74%) rename native/{furbooru_markdown => philomena_markdown}/README.md (84%) rename native/{furbooru_markdown => philomena_markdown}/src/lib.rs (89%) diff --git a/config/config.exs b/config/config.exs index 7df24019..25661e48 100644 --- a/config/config.exs +++ b/config/config.exs @@ -38,7 +38,7 @@ config :philomena, PhilomenaWeb.Endpoint, # Markdown config :philomena, Philomena.Markdown, - crate: "furbooru_markdown", + crate: "philomena_markdown", mode: :release config :phoenix, :template_engines, diff --git a/mix.exs b/mix.exs index 2b5f1299..4d3864fa 100644 --- a/mix.exs +++ b/mix.exs @@ -12,7 +12,7 @@ defmodule Philomena.MixProject do aliases: aliases(), deps: deps(), dialyzer: [plt_add_apps: [:mix]], - rustler_crates: [furbooru_markdown: []] + rustler_crates: [philomena_markdown: []] ] end diff --git a/mix.lock b/mix.lock index 8ccf9143..2facfecf 100644 --- a/mix.lock +++ b/mix.lock @@ -40,7 +40,7 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mint": {:hex, :mint, "1.3.0", "396b3301102f7b775e103da5a20494b25753aed818d6d6f0ad222a3a018c3600", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "a9aac960562e43ca69a77e5176576abfa78b8398cec5543dd4fb4ab0131d5c1e"}, + "mint": {:hex, :mint, "1.4.0", "cd7d2451b201fc8e4a8fd86257fb3878d9e3752899eb67b0c5b25b180bde1212", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "10a99e144b815cbf8522dccbc8199d15802440fc7a64d67b6853adb6fa170217"}, "mix_audit": {:hex, :mix_audit, "0.1.4", "35c424173a574436a80ad7f63cf014a7d9ce727de8cd4e7b4138d90b11aec043", [:make, :mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.4.0", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "7a43fee661bcadbad31aa04a86d33a890421c174723814b8a3a7f0e7076936a1"}, "neotoma": {:hex, :neotoma, "1.7.3", "d8bd5404b73273989946e4f4f6d529e5c2088f5fa1ca790b4dbe81f4be408e61", [:rebar], [], "hexpm", "2da322b9b1567ffa0706a7f30f6bbbde70835ae44a1050615f4b4a3d436e0f28"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, diff --git a/native/furbooru_markdown/.cargo/config b/native/philomena_markdown/.cargo/config similarity index 100% rename from native/furbooru_markdown/.cargo/config rename to native/philomena_markdown/.cargo/config diff --git a/native/furbooru_markdown/Cargo.lock b/native/philomena_markdown/Cargo.lock similarity index 99% rename from native/furbooru_markdown/Cargo.lock rename to native/philomena_markdown/Cargo.lock index 10243927..1b8087b9 100644 --- a/native/furbooru_markdown/Cargo.lock +++ b/native/philomena_markdown/Cargo.lock @@ -137,12 +137,11 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/furbooru/comrak#b69625ef55b855675d9597c9dd5fe2dda50cc0c6" +source = "git+https://github.com/philomena-dev/comrak?branch=main#94941ad6f41fa308383601e5b098c643e3bf80e2" dependencies = [ "clap", "entities", "lazy_static", - "log", "pest", "pest_derive", "regex", @@ -221,19 +220,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" -[[package]] -name = "furbooru_markdown" -version = "0.2.0" -dependencies = [ - "comrak", - "jemallocator", - "lazy_static", - "log", - "pretty_env_logger", - "regex", - "rustler", -] - [[package]] name = "generic-array" version = "0.12.4" @@ -461,6 +447,19 @@ dependencies = [ "sha-1", ] +[[package]] +name = "philomena_markdown" +version = "0.2.0" +dependencies = [ + "comrak", + "jemallocator", + "lazy_static", + "log", + "pretty_env_logger", + "regex", + "rustler", +] + [[package]] name = "pkg-config" version = "0.3.19" diff --git a/native/furbooru_markdown/Cargo.toml b/native/philomena_markdown/Cargo.toml similarity index 74% rename from native/furbooru_markdown/Cargo.toml rename to native/philomena_markdown/Cargo.toml index 66ba6392..fc691309 100644 --- a/native/furbooru_markdown/Cargo.toml +++ b/native/philomena_markdown/Cargo.toml @@ -1,16 +1,16 @@ [package] -name = "furbooru_markdown" +name = "philomena_markdown" version = "0.2.0" authors = ["Xe ", "Luna ", "Liam White "] edition = "2018" [lib] -name = "furbooru_markdown" +name = "philomena_markdown" path = "src/lib.rs" crate-type = ["dylib"] [dependencies] -comrak = { git = "https://github.com/furbooru/comrak" } +comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main" } lazy_static = "1.4" regex = "1.5" log = "0" diff --git a/native/furbooru_markdown/README.md b/native/philomena_markdown/README.md similarity index 84% rename from native/furbooru_markdown/README.md rename to native/philomena_markdown/README.md index e884ccc5..803fb867 100644 --- a/native/furbooru_markdown/README.md +++ b/native/philomena_markdown/README.md @@ -1,4 +1,4 @@ -# NIF for Elixir.Furbooru.Markdown +# NIF for Elixir.Philomena.Markdown ## To build the NIF module: @@ -9,8 +9,8 @@ ## To load the NIF: ```elixir -defmodule Furbooru.Markdown do - use Rustler, otp_app: , crate: "furbooru_markdown" +defmodule Philomena.Markdown do + use Rustler, otp_app: , crate: "philomena_markdown" # When your NIF is loaded, it will override this function. def add(_a, _b), do: :erlang.nif_error(:nif_not_loaded) diff --git a/native/furbooru_markdown/src/lib.rs b/native/philomena_markdown/src/lib.rs similarity index 89% rename from native/furbooru_markdown/src/lib.rs rename to native/philomena_markdown/src/lib.rs index 466439af..fed1baae 100644 --- a/native/furbooru_markdown/src/lib.rs +++ b/native/philomena_markdown/src/lib.rs @@ -38,11 +38,8 @@ fn to_html(input: String) -> String { options.extension.table = true; options.extension.description_lists = true; options.extension.superscript = true; - options.extension.subscript = true; - options.extension.spoiler = true; options.extension.strikethrough = true; - options.extension.underline = true; - // options.extension.furbooru = true; + options.extension.philomena = true; options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; @@ -82,11 +79,8 @@ fn to_html_unsafe(input: String) -> String { options.extension.table = true; options.extension.description_lists = true; options.extension.superscript = true; - options.extension.subscript = true; - options.extension.spoiler = true; options.extension.strikethrough = true; - options.extension.underline = true; - // options.extension.furbooru = true; + options.extension.philomena = true; options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; From f66b5599faa5eb39a23c94398af98e2a946dd364 Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 13 Sep 2021 00:38:46 +0200 Subject: [PATCH 10/25] clean up --- native/philomena_markdown/Cargo.lock | 158 ++++++++++++++++----------- native/philomena_markdown/Cargo.toml | 7 +- native/philomena_markdown/src/lib.rs | 84 +++----------- 3 files changed, 115 insertions(+), 134 deletions(-) diff --git a/native/philomena_markdown/Cargo.lock b/native/philomena_markdown/Cargo.lock index 1b8087b9..dc8a70b6 100644 --- a/native/philomena_markdown/Cargo.lock +++ b/native/philomena_markdown/Cargo.lock @@ -177,19 +177,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -220,6 +207,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "generic-array" version = "0.12.4" @@ -253,15 +252,6 @@ dependencies = [ "libc", ] -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "indexmap" version = "1.7.0" @@ -326,15 +316,6 @@ dependencies = [ "safemem", ] -[[package]] -name = "log" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" -dependencies = [ - "cfg-if", -] - [[package]] name = "maplit" version = "1.0.2" @@ -449,14 +430,11 @@ dependencies = [ [[package]] name = "philomena_markdown" -version = "0.2.0" +version = "0.3.0" dependencies = [ "comrak", "jemallocator", - "lazy_static", - "log", - "pretty_env_logger", - "regex", + "rust-crypto", "rustler", ] @@ -480,16 +458,6 @@ dependencies = [ "xml-rs", ] -[[package]] -name = "pretty_env_logger" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" -dependencies = [ - "env_logger", - "log", -] - [[package]] name = "proc-macro2" version = "1.0.29" @@ -499,12 +467,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "1.0.9" @@ -514,6 +476,53 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "regex" version = "1.5.4" @@ -531,6 +540,25 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "rust-crypto" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +dependencies = [ + "gcc", + "libc", + "rand 0.3.23", + "rustc-serialize", + "time", +] + +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" + [[package]] name = "rustler" version = "0.22.0" @@ -668,15 +696,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "termcolor" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" -dependencies = [ - "winapi-util", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -686,6 +705,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + [[package]] name = "twoway" version = "0.2.2" @@ -776,6 +806,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "winapi" version = "0.3.9" diff --git a/native/philomena_markdown/Cargo.toml b/native/philomena_markdown/Cargo.toml index fc691309..a0460a7d 100644 --- a/native/philomena_markdown/Cargo.toml +++ b/native/philomena_markdown/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "philomena_markdown" -version = "0.2.0" +version = "0.3.0" authors = ["Xe ", "Luna ", "Liam White "] edition = "2018" @@ -11,12 +11,9 @@ crate-type = ["dylib"] [dependencies] comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main" } -lazy_static = "1.4" -regex = "1.5" -log = "0" -pretty_env_logger = "0" rustler = "0.22" jemallocator = "0.3.2" +rust-crypto = "0.2" [profile.release] opt-level = 3 diff --git a/native/philomena_markdown/src/lib.rs b/native/philomena_markdown/src/lib.rs index fed1baae..e8fdcfab 100644 --- a/native/philomena_markdown/src/lib.rs +++ b/native/philomena_markdown/src/lib.rs @@ -1,92 +1,40 @@ use comrak::{markdown_to_html, ComrakOptions}; -use lazy_static::lazy_static; -use regex::{Captures, Regex}; -use rustler::Term; use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -mod atoms { - rustler::atoms! { - ok, - } -} - rustler::init! { "Elixir.Philomena.Markdown", [to_html, to_html_unsafe] } -const IMAGE_MENTION_REGEX: &'static str = r#">>([0-9]+)([|t|s|p]?)"#; - -lazy_static! { - static ref IMAGE_MENTION_REPLACE: Regex = Regex::new(IMAGE_MENTION_REGEX).unwrap(); +fn common_options() -> ComrakOptions { + let mut options = ComrakOptions::default(); + options.extension.autolink = true; + options.extension.table = true; + options.extension.description_lists = true; + options.extension.superscript = true; + options.extension.strikethrough = true; + options.extension.philomena = true; + options.parse.smart = true; + options.render.hardbreaks = true; + options.render.github_pre_lang = true; + options } #[rustler::nif(schedule = "DirtyCpu")] fn to_html(input: String) -> String { - let _ = pretty_env_logger::try_init(); - let mut text: String = input; - - if text.contains(">>") { - text = text.replace(">>", ">>"); - } - - let mut options = ComrakOptions::default(); - options.extension.autolink = true; - options.extension.table = true; - options.extension.description_lists = true; - options.extension.superscript = true; - options.extension.strikethrough = true; - options.extension.philomena = true; - options.parse.smart = true; - options.render.hardbreaks = true; - options.render.github_pre_lang = true; + let mut options = common_options(); options.render.escape = true; - let mut result = markdown_to_html(&text, &options); - result = match IMAGE_MENTION_REPLACE.captures(&result) { - None => result, - Some(fields) => { - match fields.get(2).unwrap().as_str() { - "t" => result, // TODO(Xe): thumbnail rendering - "s" => result, // TODO(Xe): small preview rendering - "p" => result, // TODO(Xe): large preview rendering - "" => String::from( - IMAGE_MENTION_REPLACE.replace_all(&result, |caps: &Captures| { - format!(r#">>{0}"#, &caps[1]) - }), - ), - _ => result, - } - } - }; - - result + markdown_to_html(&input, &options) } #[rustler::nif(schedule = "DirtyCpu")] fn to_html_unsafe(input: String) -> String { - let mut text: String = input; - - if text.contains(">>") { - text = text.replace(">>", ">>"); - } - - let mut options = ComrakOptions::default(); - options.extension.autolink = true; - options.extension.table = true; - options.extension.description_lists = true; - options.extension.superscript = true; - options.extension.strikethrough = true; - options.extension.philomena = true; - options.parse.smart = true; - options.render.hardbreaks = true; - options.render.github_pre_lang = true; + let mut options = common_options(); options.render.unsafe_ = true; - let result = markdown_to_html(&text, &options); - - result + markdown_to_html(&input, &options) } From 758c306036430b553b415b8d5f976c3e4783c1a1 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Sun, 12 Sep 2021 18:50:42 -0400 Subject: [PATCH 11/25] scrap layer caching --- .github/workflows/elixir.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index c3b9c241..1aa794cf 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -16,9 +16,7 @@ jobs: key: ${{ runner.os }}-build-deps-${{ hashFiles('mix.lock') }} - run: docker-compose pull - - - uses: satackey/action-docker-layer-caching@v0.0.8 - continue-on-error: true + - run: docker-compose build - name: Build and test run: docker-compose run app run-test @@ -27,8 +25,3 @@ jobs: run: | docker-compose run app mix sobelow --config docker-compose run app mix deps.audit - - - name: Fix permissions - run: | - sudo chown -R $(whoami):$(id -ng) deps - sudo chown -R $(whoami):$(id -ng) _build From 1bf7c647a1dbbf8755f4b41fd5df61105dcef6ee Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Sun, 12 Sep 2021 19:19:31 -0400 Subject: [PATCH 12/25] Rename crate to philomena and change nif mod to Philomena.Native --- config/config.exs | 4 ++-- lib/philomena/markdown.ex | 6 ++---- lib/philomena/native.ex | 7 +++++++ .../{philomena_markdown => philomena}/Cargo.lock | 2 +- .../{philomena_markdown => philomena}/Cargo.toml | 4 ++-- native/{philomena_markdown => philomena}/README.md | 0 .../{philomena_markdown => philomena}/src/lib.rs | 14 +++++++------- native/philomena_markdown/.cargo/config | 5 ----- 8 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 lib/philomena/native.ex rename native/{philomena_markdown => philomena}/Cargo.lock (99%) rename native/{philomena_markdown => philomena}/Cargo.toml (88%) rename native/{philomena_markdown => philomena}/README.md (100%) rename native/{philomena_markdown => philomena}/src/lib.rs (72%) delete mode 100644 native/philomena_markdown/.cargo/config diff --git a/config/config.exs b/config/config.exs index 25661e48..2c797a1a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -37,8 +37,8 @@ config :philomena, PhilomenaWeb.Endpoint, pubsub_server: Philomena.PubSub # Markdown -config :philomena, Philomena.Markdown, - crate: "philomena_markdown", +config :philomena, Philomena.Native, + crate: "philomena", mode: :release config :phoenix, :template_engines, diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index d25cac8d..fbff2a48 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -1,11 +1,9 @@ defmodule Philomena.Markdown do - use Rustler, otp_app: :philomena - @markdown_chars ~r/[\*_\[\]\(\)\^`\%\\~<>#\|]/ # When your NIF is loaded, it will override this function. - def to_html(_text), do: :erlang.nif_error(:nif_not_loaded) - def to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) + def to_html(text), do: Philomena.Native.markdown_to_html(text) + def to_html_unsafe(text), do: Philomena.Native.markdown_to_html_unsafe(text) def escape_markdown(text) do @markdown_chars diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex new file mode 100644 index 00000000..b5950029 --- /dev/null +++ b/lib/philomena/native.ex @@ -0,0 +1,7 @@ +defmodule Philomena.Native do + use Rustler, otp_app: :philomena + + # When your NIF is loaded, it will override this function. + def markdown_to_html(_text), do: :erlang.nif_error(:nif_not_loaded) + def markdown_to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) +end diff --git a/native/philomena_markdown/Cargo.lock b/native/philomena/Cargo.lock similarity index 99% rename from native/philomena_markdown/Cargo.lock rename to native/philomena/Cargo.lock index dc8a70b6..f808352d 100644 --- a/native/philomena_markdown/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -429,7 +429,7 @@ dependencies = [ ] [[package]] -name = "philomena_markdown" +name = "philomena" version = "0.3.0" dependencies = [ "comrak", diff --git a/native/philomena_markdown/Cargo.toml b/native/philomena/Cargo.toml similarity index 88% rename from native/philomena_markdown/Cargo.toml rename to native/philomena/Cargo.toml index a0460a7d..2ef3e497 100644 --- a/native/philomena_markdown/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "philomena_markdown" +name = "philomena" version = "0.3.0" authors = ["Xe ", "Luna ", "Liam White "] edition = "2018" [lib] -name = "philomena_markdown" +name = "philomena" path = "src/lib.rs" crate-type = ["dylib"] diff --git a/native/philomena_markdown/README.md b/native/philomena/README.md similarity index 100% rename from native/philomena_markdown/README.md rename to native/philomena/README.md diff --git a/native/philomena_markdown/src/lib.rs b/native/philomena/src/lib.rs similarity index 72% rename from native/philomena_markdown/src/lib.rs rename to native/philomena/src/lib.rs index e8fdcfab..03bcf8b1 100644 --- a/native/philomena_markdown/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,12 +1,12 @@ -use comrak::{markdown_to_html, ComrakOptions}; +use comrak::ComrakOptions; use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; rustler::init! { - "Elixir.Philomena.Markdown", - [to_html, to_html_unsafe] + "Elixir.Philomena.Native", + [markdown_to_html, markdown_to_html_unsafe] } fn common_options() -> ComrakOptions { @@ -24,17 +24,17 @@ fn common_options() -> ComrakOptions { } #[rustler::nif(schedule = "DirtyCpu")] -fn to_html(input: String) -> String { +fn markdown_to_html(input: String) -> String { let mut options = common_options(); options.render.escape = true; - markdown_to_html(&input, &options) + comrak::markdown_to_html(&input, &options) } #[rustler::nif(schedule = "DirtyCpu")] -fn to_html_unsafe(input: String) -> String { +fn markdown_to_html_unsafe(input: String) -> String { let mut options = common_options(); options.render.unsafe_ = true; - markdown_to_html(&input, &options) + comrak::markdown_to_html(&input, &options) } diff --git a/native/philomena_markdown/.cargo/config b/native/philomena_markdown/.cargo/config deleted file mode 100644 index a125291f..00000000 --- a/native/philomena_markdown/.cargo/config +++ /dev/null @@ -1,5 +0,0 @@ -[target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] From 6aa9baefd23e76ed79aba850c27c1a7a15952f51 Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 13 Sep 2021 20:54:57 +0200 Subject: [PATCH 13/25] implement camo in rust --- .gitignore | 3 + lib/camo/image.ex | 39 +---- lib/philomena/native.ex | 5 +- native/philomena/Cargo.lock | 273 +++++++++++++++++++++---------- native/philomena/Cargo.toml | 4 +- native/philomena/shell.nix | 11 ++ native/philomena/src/camo.rs | 46 ++++++ native/philomena/src/lib.rs | 37 ++--- native/philomena/src/markdown.rs | 29 ++++ 9 files changed, 302 insertions(+), 145 deletions(-) create mode 100644 native/philomena/shell.nix create mode 100644 native/philomena/src/camo.rs create mode 100644 native/philomena/src/markdown.rs diff --git a/.gitignore b/.gitignore index fcadf67e..0caa7395 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ npm-debug.log # Unportable compiled binaries /priv/native/ + +# Rust binaries +/native/**/target diff --git a/lib/camo/image.ex b/lib/camo/image.ex index 43152d7a..a2d49655 100644 --- a/lib/camo/image.ex +++ b/lib/camo/image.ex @@ -1,40 +1,3 @@ defmodule Camo.Image do - def image_url(input) do - uri = URI.parse(input) - - cond do - is_nil(uri.host) -> - "" - - is_nil(camo_key()) -> - input - - uri.host in [cdn_host(), camo_host()] -> - URI.to_string(%{uri | scheme: "https", port: 443}) - - true -> - camo_digest = :crypto.mac(:hmac, :sha, camo_key(), input) |> Base.encode16(case: :lower) - - camo_uri = %URI{ - host: camo_host(), - path: "/" <> camo_digest, - query: URI.encode_query(url: input), - scheme: "https" - } - - URI.to_string(camo_uri) - end - end - - defp cdn_host do - Application.get_env(:philomena, :cdn_host) - end - - defp camo_key do - Application.get_env(:philomena, :camo_key) - end - - defp camo_host do - Application.get_env(:philomena, :camo_host) - end + def image_url(input), do: Philomena.Native.camo_image_url(input) end diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index b5950029..4d9f5ebf 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -1,7 +1,10 @@ defmodule Philomena.Native do use Rustler, otp_app: :philomena - # When your NIF is loaded, it will override this function. + # Markdown def markdown_to_html(_text), do: :erlang.nif_error(:nif_not_loaded) def markdown_to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) + + # Camo + def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index f808352d..817d2379 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -43,6 +43,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base16" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27c3610c36aee21ce8ac510e6224498de4228ad772a171ed65643a24693a5a8" + [[package]] name = "base64" version = "0.13.0" @@ -85,6 +91,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bumpalo" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" + [[package]] name = "byte-tools" version = "0.3.1" @@ -201,24 +213,22 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "generic-array" version = "0.12.4" @@ -252,6 +262,17 @@ dependencies = [ "libc", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -289,6 +310,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1866b355d9c878e5e607473cbe3f63282c0b7aad2db1dbebf55076c686918254" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -316,12 +346,27 @@ dependencies = [ "safemem", ] +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.4.1" @@ -357,6 +402,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + [[package]] name = "onig" version = "6.2.0" @@ -385,6 +436,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "pest" version = "2.1.3" @@ -432,10 +489,12 @@ dependencies = [ name = "philomena" version = "0.3.0" dependencies = [ + "base16", "comrak", "jemallocator", - "rust-crypto", + "ring", "rustler", + "url", ] [[package]] @@ -476,53 +535,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" -dependencies = [ - "libc", - "rand 0.4.6", -] - -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "regex" version = "1.5.4" @@ -541,24 +553,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] -name = "rust-crypto" -version = "0.2.36" +name = "ring" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ - "gcc", + "cc", "libc", - "rand 0.3.23", - "rustc-serialize", - "time", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", ] -[[package]] -name = "rustc-serialize" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" - [[package]] name = "rustler" version = "0.22.0" @@ -658,6 +666,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074" +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strsim" version = "0.8.0" @@ -706,16 +720,20 @@ dependencies = [ ] [[package]] -name = "time" -version = "0.1.44" +name = "tinyvec" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "5241dd6f21443a3606b432718b166d3cedc962fd4b8bea54a8bc7f514ebda986" dependencies = [ - "libc", - "wasi", - "winapi", + "tinyvec_macros", ] +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "twoway" version = "0.2.2" @@ -750,6 +768,21 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" +[[package]] +name = "unicode-bidi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -783,6 +816,24 @@ dependencies = [ "void", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "vec_map" version = "0.8.2" @@ -807,10 +858,68 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +name = "wasm-bindgen" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d5a6580be83b19dc570a8f9c324251687ab2184e57086f71625feb57ec77c8" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3775a030dc6f5a0afd8a84981a21cc92a781eb429acef9ecce476d0c9113e92" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" + +[[package]] +name = "web-sys" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a84d70d1ec7d2da2d26a5bd78f4bca1b8c3254805363ce743b7a05bc30d195a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "winapi" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 2ef3e497..2fb77ea9 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -13,7 +13,9 @@ crate-type = ["dylib"] comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main" } rustler = "0.22" jemallocator = "0.3.2" -rust-crypto = "0.2" +ring = "0.16" +base16 = "0.2" +url = "2.2" [profile.release] opt-level = 3 diff --git a/native/philomena/shell.nix b/native/philomena/shell.nix new file mode 100644 index 00000000..92bfa4f6 --- /dev/null +++ b/native/philomena/shell.nix @@ -0,0 +1,11 @@ +let + pkgs = import {}; +in pkgs.mkShell { + buildInputs = [ + pkgs.cargo + pkgs.rustc + pkgs.rustfmt + pkgs.clippy + pkgs.rust-analyzer + ]; +} diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs new file mode 100644 index 00000000..cdd4d041 --- /dev/null +++ b/native/philomena/src/camo.rs @@ -0,0 +1,46 @@ +use ring::hmac; +use std::env; +use url::Url; + +fn trusted_host(mut url: Url) -> Option { + url.set_port(Some(443)).ok()?; + url.set_scheme("https").ok()?; + + Some(url.to_string()) +} + +fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option { + let camo_url = format!("https://{}", camo_host); + let key = hmac::Key::new(hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, camo_key.as_ref()); + let tag = hmac::sign(&key, url.to_string().as_bytes()); + let encoded = base16::encode_lower(tag.as_ref()); + + let mut camo_uri = Url::parse(&camo_url).ok()?; + camo_uri.set_path(&encoded); + camo_uri.set_port(Some(443)).ok()?; + camo_uri.set_scheme("https").ok()?; + camo_uri + .query_pairs_mut() + .clear() + .append_pair("url", &url.to_string()); + + Some(camo_uri.to_string()) +} + +pub fn image_url(uri: String) -> Option { + let cdn_host = env::var("CDN_HOST").ok()?; + let camo_host = env::var("CAMO_HOST").ok()?; + let camo_key = env::var("CAMO_KEY").ok()?; + + if camo_key.is_empty() { + return Some(uri); + } + + let url = Url::parse(&uri).ok()?; + + match url.host_str() { + Some(hostname) if hostname == cdn_host || hostname == camo_host => trusted_host(url), + Some(_) => untrusted_host(url, camo_host, camo_key), + None => Some(String::from("")), + } +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 03bcf8b1..6cc393f6 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,40 +1,31 @@ -use comrak::ComrakOptions; use jemallocator::Jemalloc; +mod camo; +mod markdown; + #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; rustler::init! { "Elixir.Philomena.Native", - [markdown_to_html, markdown_to_html_unsafe] + [markdown_to_html, markdown_to_html_unsafe, camo_image_url] } -fn common_options() -> ComrakOptions { - let mut options = ComrakOptions::default(); - options.extension.autolink = true; - options.extension.table = true; - options.extension.description_lists = true; - options.extension.superscript = true; - options.extension.strikethrough = true; - options.extension.philomena = true; - options.parse.smart = true; - options.render.hardbreaks = true; - options.render.github_pre_lang = true; - options -} +// Markdown NIF wrappers. #[rustler::nif(schedule = "DirtyCpu")] fn markdown_to_html(input: String) -> String { - let mut options = common_options(); - options.render.escape = true; - - comrak::markdown_to_html(&input, &options) + markdown::to_html(input) } #[rustler::nif(schedule = "DirtyCpu")] fn markdown_to_html_unsafe(input: String) -> String { - let mut options = common_options(); - options.render.unsafe_ = true; - - comrak::markdown_to_html(&input, &options) + markdown::to_html_unsafe(input) +} + +// Camo NIF wrappers. + +#[rustler::nif] +fn camo_image_url(input: String) -> String { + camo::image_url(input).unwrap_or_else(|| String::from("")) } diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs new file mode 100644 index 00000000..41fcac45 --- /dev/null +++ b/native/philomena/src/markdown.rs @@ -0,0 +1,29 @@ +use comrak::ComrakOptions; + +fn common_options() -> ComrakOptions { + let mut options = ComrakOptions::default(); + options.extension.autolink = true; + options.extension.table = true; + options.extension.description_lists = true; + options.extension.superscript = true; + options.extension.strikethrough = true; + options.extension.philomena = true; + options.parse.smart = true; + options.render.hardbreaks = true; + options.render.github_pre_lang = true; + options +} + +pub fn to_html(input: String) -> String { + let mut options = common_options(); + options.render.escape = true; + + comrak::markdown_to_html(&input, &options) +} + +pub fn to_html_unsafe(input: String) -> String { + let mut options = common_options(); + options.render.unsafe_ = true; + + comrak::markdown_to_html(&input, &options) +} From 670242bc1376ce1cb3b3afcbffd7e4aedbe0fda2 Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 13 Sep 2021 21:55:42 +0200 Subject: [PATCH 14/25] camoifier in markdown --- native/philomena/Cargo.lock | 6 +++--- native/philomena/shell.nix | 11 ----------- native/philomena/src/markdown.rs | 4 ++++ 3 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 native/philomena/shell.nix diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 817d2379..d38cfcf4 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#94941ad6f41fa308383601e5b098c643e3bf80e2" +source = "git+https://github.com/philomena-dev/comrak?branch=main#a4da46ece62bc99a82cfffb37fa7798d1b1da132" dependencies = [ "clap", "entities", @@ -197,9 +197,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80edafed416a46fb378521624fab1cfa2eb514784fd8921adbe8a8d8321da811" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" dependencies = [ "cfg-if", "crc32fast", diff --git a/native/philomena/shell.nix b/native/philomena/shell.nix deleted file mode 100644 index 92bfa4f6..00000000 --- a/native/philomena/shell.nix +++ /dev/null @@ -1,11 +0,0 @@ -let - pkgs = import {}; -in pkgs.mkShell { - buildInputs = [ - pkgs.cargo - pkgs.rustc - pkgs.rustfmt - pkgs.clippy - pkgs.rust-analyzer - ]; -} diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index 41fcac45..af74785a 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,4 +1,5 @@ use comrak::ComrakOptions; +use crate::camo; fn common_options() -> ComrakOptions { let mut options = ComrakOptions::default(); @@ -11,6 +12,9 @@ fn common_options() -> ComrakOptions { options.parse.smart = true; options.render.hardbreaks = true; options.render.github_pre_lang = true; + + options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); + options } From 8f6f56b0e7c54a3f1bc40a441a6687058cc3f409 Mon Sep 17 00:00:00 2001 From: Luna D Date: Tue, 14 Sep 2021 01:08:47 +0200 Subject: [PATCH 15/25] your mom's instrument --- lib/philomena_web/textile_markdown_renderer.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/philomena_web/textile_markdown_renderer.ex b/lib/philomena_web/textile_markdown_renderer.ex index 43d34787..04c500f8 100644 --- a/lib/philomena_web/textile_markdown_renderer.ex +++ b/lib/philomena_web/textile_markdown_renderer.ex @@ -1,11 +1,11 @@ defmodule PhilomenaWeb.TextileMarkdownRenderer do alias Philomena.Textile.ParserMarkdown - def render_one(post, conn) do - hd(render_collection([post], conn)) + def render_one(post) do + hd(render_collection([post])) end - def render_collection(posts, conn) do + def render_collection(posts) do opts = %{image_transform: &Camo.Image.image_url/1} parsed = Enum.map(posts, &ParserMarkdown.parse(opts, &1.body)) From ece485b367c57b4df6290ca8079605405517ad75 Mon Sep 17 00:00:00 2001 From: Luna D Date: Tue, 14 Sep 2021 23:41:46 +0200 Subject: [PATCH 16/25] fix bq --- lib/philomena/textile/parser_markdown.ex | 149 +++++++++--------- .../controllers/post/preview_controller.ex | 2 +- lib/philomena_web/text_renderer.ex | 5 +- 3 files changed, 82 insertions(+), 74 deletions(-) diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 20bdd371..62b4a65c 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -25,7 +25,7 @@ defmodule Philomena.Textile.ParserMarkdown do parser = Map.put(parser, :state, %{}) with {:ok, tokens, _1, _2, _3, _4} <- Lexer.lex(String.trim(input || "")), - {:ok, tree, []} <- repeat(&textile/2, parser, tokens) do + {:ok, tree, []} <- repeat(&textile/3, parser, tokens, false, 0) do partial_flatten(tree) else _ -> @@ -68,21 +68,21 @@ defmodule Philomena.Textile.ParserMarkdown do # Helper corresponding to Kleene star (*) operator # Match a specificed rule zero or more times - defp repeat(rule, parser, tokens, bq) when bq == true do - case rule.(parser, tokens, true) do + defp repeat(rule, parser, tokens, bq, level) when bq == true do + case rule.(parser, tokens, true, level) do {:ok, tree, r_tokens} -> - {:ok, tree2, r2_tokens} = repeat(rule, parser, r_tokens, true) - {:ok, [tree, tree2], r2_tokens} + {:ok, tree2, r2_tokens, level} = repeat(rule, parser, r_tokens, true, level) + {:ok, [tree, tree2], r2_tokens, level} _ -> - {:ok, [], tokens} + {:ok, [], tokens, level} end end - defp repeat(rule, parser, tokens) do - case rule.(parser, tokens) do + defp repeat(rule, parser, tokens, bq, level) do + case rule.(parser, tokens, level) do {:ok, tree, r_tokens} -> - {:ok, tree2, r2_tokens} = repeat(rule, parser, r_tokens) + {:ok, tree2, r2_tokens} = repeat(rule, parser, r_tokens, false, level) {:ok, [tree, tree2], r2_tokens} _ -> @@ -96,20 +96,20 @@ defmodule Philomena.Textile.ParserMarkdown do # defp simple_recursive(:bq_open, :bq_close, open_tag, close_tag, callback, parser, [ {:bq_open, open} | r_tokens - ]) do - case repeat(callback, parser, r_tokens, true) do - {:ok, tree, [{:bq_close, _} | r2_tokens]} -> - {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} + ], level) do + case repeat(callback, parser, r_tokens, true, level) do + {:ok, tree, [{:bq_close, _} | r2_tokens], level} -> + {:ok, [{:markup, "\n" <> String.duplicate(open_tag, level)}, tree, {:markup, "\n" <> String.duplicate(close_tag, level - 1)}], r2_tokens} - {:ok, tree, r2_tokens} -> + {:ok, tree, r2_tokens, level} -> {:ok, [{:text, escape(open)}, tree], r2_tokens} end end defp simple_recursive(open_token, close_token, open_tag, close_tag, callback, parser, [ {open_token, open} | r_tokens - ]) do - case repeat(callback, parser, r_tokens) do + ], level) do + case repeat(callback, parser, r_tokens, false, level) do {:ok, tree, [{^close_token, _} | r2_tokens]} -> {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} @@ -125,7 +125,8 @@ defmodule Philomena.Textile.ParserMarkdown do _close_tag, _callback, _parser, - _tokens + _tokens, + _level ) do {:error, "Expected a simple recursive rule"} end @@ -143,7 +144,8 @@ defmodule Philomena.Textile.ParserMarkdown do callback, state, parser, - [{open_token, open} | r_tokens] + [{open_token, open} | r_tokens], + level ) do case parser.state do %{^state => _} -> @@ -155,7 +157,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:text, escape(open)}], r_tokens} _ -> - case repeat(callback, put_state(parser, state), r_tokens) do + case repeat(callback, put_state(parser, state), r_tokens, false, level) do {:ok, tree, [{^close_token, close}, {^lookahead_not, ln} | r2_tokens]} -> {:ok, [{:text, escape(open)}, tree, {:text, escape(close)}], [{lookahead_not, ln} | r2_tokens]} @@ -179,7 +181,8 @@ defmodule Philomena.Textile.ParserMarkdown do _callback, _state, _parser, - _tokens + _tokens, + _level ) do {:error, "Expected a simple lookahead not rule"} end @@ -202,7 +205,7 @@ defmodule Philomena.Textile.ParserMarkdown do # inline_textile_element_not_opening_markup; # - defp inline_textile_element(parser, tokens) do + defp inline_textile_element(parser, tokens, level) do [ {:b_delim, :b, "**", "**"}, {:i_delim, :i, "_", "_"}, @@ -220,10 +223,11 @@ defmodule Philomena.Textile.ParserMarkdown do open_tag, close_tag, :quicktxt, - &inline_textile_element/2, + &inline_textile_element/3, state, parser, - tokens + tokens, + level ) |> case do {:ok, tree, r_tokens} -> @@ -234,12 +238,12 @@ defmodule Philomena.Textile.ParserMarkdown do end end) |> case do - nil -> inner_inline_textile_element(parser, tokens) + nil -> inner_inline_textile_element(parser, tokens, level) value -> value end end - defp inner_inline_textile_element(parser, [{token, t}, {:quicktxt, q} | r_tokens]) + defp inner_inline_textile_element(parser, [{token, t}, {:quicktxt, q} | r_tokens], level) when token in [ :b_delim, :i_delim, @@ -250,7 +254,7 @@ defmodule Philomena.Textile.ParserMarkdown do :del_delim, :sub_delim ] do - case inline_textile_element(parser, [{:quicktxt, q} | r_tokens]) do + case inline_textile_element(parser, [{:quicktxt, q} | r_tokens], level) do {:ok, tree, r2_tokens} -> {:ok, [{:text, escape(t)}, tree], r2_tokens} @@ -259,8 +263,8 @@ defmodule Philomena.Textile.ParserMarkdown do end end - defp inner_inline_textile_element(parser, [{:link_delim, open} | r_tokens]) do - case repeat(&block_textile_element/2, parser, r_tokens) do + defp inner_inline_textile_element(parser, [{:link_delim, open} | r_tokens], level) do + case repeat(&block_textile_element/3, parser, r_tokens, false, level) do {:ok, tree, [{:unbracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> href = escape(url) @@ -273,8 +277,8 @@ defmodule Philomena.Textile.ParserMarkdown do end end - defp inner_inline_textile_element(parser, [{:bracketed_link_open, open} | r_tokens]) do - case repeat(&inline_textile_element/2, parser, r_tokens) do + defp inner_inline_textile_element(parser, [{:bracketed_link_open, open} | r_tokens], level) do + case repeat(&inline_textile_element/3, parser, r_tokens, false, level) do {:ok, tree, [{:bracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> href = escape(url) @@ -290,7 +294,7 @@ defmodule Philomena.Textile.ParserMarkdown do defp inner_inline_textile_element(parser, [ {token, img}, {:unbracketed_image_url, <<":", url::binary>>} | r_tokens - ]) + ], level) when token in [:unbracketed_image, :bracketed_image] do img = parser.image_transform.(img) @@ -304,7 +308,7 @@ defmodule Philomena.Textile.ParserMarkdown do ], r_tokens} end - defp inner_inline_textile_element(parser, [{token, img} | r_tokens]) + defp inner_inline_textile_element(parser, [{token, img} | r_tokens], level) when token in [:unbracketed_image, :bracketed_image] do img = parser.image_transform.(img) @@ -316,13 +320,13 @@ defmodule Philomena.Textile.ParserMarkdown do ], r_tokens} end - defp inner_inline_textile_element(parser, [{:code_delim, open} | r_tokens]) do + defp inner_inline_textile_element(parser, [{:code_delim, open} | r_tokens], level) do case parser.state do %{code: _} -> {:error, "End of rule"} _ -> - case repeat(&inline_textile_element/2, put_state(parser, :code), r_tokens) do + case repeat(&inline_textile_element/3, put_state(parser, :code), r_tokens, false, level) do {:ok, tree, [{:code_delim, _} | r2_tokens]} -> {:ok, [{:markup, "`"}, tree, {:markup, "`"}], r2_tokens} @@ -332,8 +336,8 @@ defmodule Philomena.Textile.ParserMarkdown do end end - defp inner_inline_textile_element(parser, tokens) do - inline_textile_element_not_opening_markup(parser, tokens) + defp inner_inline_textile_element(parser, tokens, level \\ 0) do + inline_textile_element_not_opening_markup(parser, tokens, level) end # @@ -342,27 +346,27 @@ defmodule Philomena.Textile.ParserMarkdown do # Note that text is not escaped here because it will be escaped # when the tree is flattened - defp bq_cite_text(_parser, [{:bq_cite_open, _open} | _rest]) do + defp bq_cite_text(_parser, [{:bq_cite_open, _open} | _rest], level) do {:error, "Expected cite tokens"} end - defp bq_cite_text(_parser, [{:char, lit} | r_tokens]) do + defp bq_cite_text(_parser, [{:char, lit} | r_tokens], level) do {:ok, [{:text, <>}], r_tokens} end - defp bq_cite_text(_parser, [{:quicktxt, lit} | r_tokens]) do + defp bq_cite_text(_parser, [{:quicktxt, lit} | r_tokens], level) do {:ok, [{:text, <>}], r_tokens} end - defp bq_cite_text(_parser, [{:space, _} | r_tokens]) do + defp bq_cite_text(_parser, [{:space, _} | r_tokens], level) do {:ok, [{:text, " "}], r_tokens} end - defp bq_cite_text(_parser, [{_token, t} | r_tokens]) do + defp bq_cite_text(_parser, [{_token, t} | r_tokens], level) do {:ok, [{:text, t}], r_tokens} end - defp bq_cite_text(_parser, _tokens) do + defp bq_cite_text(_parser, _tokens, _level) do {:error, "Expected cite tokens"} end @@ -374,16 +378,16 @@ defmodule Philomena.Textile.ParserMarkdown do # opening_block_tag block_textile_element* closing_block_tag; # - defp inline_textile_element_not_opening_markup(_parser, [{:literal, lit} | r_tokens]) do + defp inline_textile_element_not_opening_markup(_parser, [{:literal, lit} | r_tokens], level) do {:ok, [{:markup, escape(Markdown.escape_markdown(lit))},], r_tokens} end - defp inline_textile_element_not_opening_markup(_parser, [{:space, _} | r_tokens]) do + defp inline_textile_element_not_opening_markup(_parser, [{:space, _} | r_tokens], level) do {:ok, [{:text, " "}], r_tokens} end - defp inline_textile_element_not_opening_markup(_parser, [{:char, lit} | r_tokens]) do + defp inline_textile_element_not_opening_markup(_parser, [{:char, lit} | r_tokens], level) do {binary, r2_tokens} = assemble_binary(:char, <>, r_tokens) # A bit of an ugly hack to force the parser to output >> instead of >> @@ -399,7 +403,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:quicktxt, q1}, {token, t}, {:quicktxt, q2} | r_tokens - ]) + ], level) when token in [ :b_delim, :i_delim, @@ -414,25 +418,25 @@ defmodule Philomena.Textile.ParserMarkdown do r_tokens} end - defp inline_textile_element_not_opening_markup(_parser, [{:quicktxt, lit} | r_tokens]) do + defp inline_textile_element_not_opening_markup(_parser, [{:quicktxt, lit} | r_tokens], level) do {:ok, [{:text, escape(<>)}], r_tokens} end - defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens]) do - case repeat(&bq_cite_text/2, parser, r_tokens) do + defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens], level) do + case repeat(&bq_cite_text/3, parser, r_tokens, false, level) do {:ok, tree, [{:bq_cite_open, open} | r2_tokens]} -> - case repeat(&block_textile_element/3, parser, r2_tokens, true) do - {:ok, tree2, [{:bq_close, _} | r3_tokens]} -> + case repeat(&block_textile_element/4, parser, r2_tokens, true, level + 1) do + {:ok, tree2, [{:bq_close, _} | r3_tokens], level} -> cite = escape(flatten(tree)) {:ok, [ - {:markup, "\n> "}, + {:markup, "\n" <> String.duplicate("> ", level)}, tree2, - {:markup, "\n"} + {:markup, "\n" <> String.duplicate("> ", level - 1)} ], r3_tokens} - {:ok, tree2, r3_tokens} -> + {:ok, tree2, r3_tokens, level} -> {:ok, [ {:text, escape(start)}, @@ -447,13 +451,13 @@ defmodule Philomena.Textile.ParserMarkdown do end end - defp inline_textile_element_not_opening_markup(_parser, [{:bq_cite_open, tok} | r_tokens]) do + defp inline_textile_element_not_opening_markup(_parser, [{:bq_cite_open, tok} | r_tokens], level) do {:ok, [{:text, escape(tok)}], r_tokens} end - defp inline_textile_element_not_opening_markup(parser, tokens) do + defp inline_textile_element_not_opening_markup(parser, tokens, level) do [ - {:bq_open, :bq_close, "\n> ", "\n"}, + {:bq_open, :bq_close, "> ", "> "}, {:spoiler_open, :spoiler_close, "||", "||"}, {:bracketed_b_open, :bracketed_b_close, "**", "**"}, {:bracketed_i_open, :bracketed_i_close, "_", "_"}, @@ -472,9 +476,10 @@ defmodule Philomena.Textile.ParserMarkdown do close_token, open_tag, close_tag, - &block_textile_element/3, + &block_textile_element/4, parser, - tokens + tokens, + level + 1 ) else simple_recursive( @@ -482,9 +487,10 @@ defmodule Philomena.Textile.ParserMarkdown do close_token, open_tag, close_tag, - &block_textile_element/2, + &block_textile_element/3, parser, - tokens + tokens, + level ) end |> case do @@ -503,16 +509,17 @@ defmodule Philomena.Textile.ParserMarkdown do # double_newline | newline | inline_textile_element; # - defp block_textile_element(_parser, [{:double_newline, _} | r_tokens], bq) when bq == true do - {:ok, [{:markup, "\n> \n> "}], r_tokens} + defp block_textile_element(_parser, [{:double_newline, _} | r_tokens], bq, level) when bq == true do + one = "\n" <> String.duplicate("> ", level) + {:ok, [{:markup, String.duplicate(one, 2)}], r_tokens} end - defp block_textile_element(_parser, [{:newline, _} | r_tokens], bq) when bq == true do - {:ok, [{:markup, "\n> "}], r_tokens} + defp block_textile_element(_parser, [{:newline, _} | r_tokens], bq, level) when bq == true do + {:ok, [{:markup, "\n" <> String.duplicate("> ", level)}], r_tokens} end - defp block_textile_element(parser, tokens, _bq) do - inline_textile_element(parser, tokens) + defp block_textile_element(parser, tokens, _bq, level) do + inline_textile_element(parser, tokens, level) end defp block_textile_element(_parser, [{:double_newline, _} | r_tokens]) do @@ -523,8 +530,8 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:markup, "\n"}], r_tokens} end - defp block_textile_element(parser, tokens) do - inline_textile_element(parser, tokens) + defp block_textile_element(parser, tokens, level) do + inline_textile_element(parser, tokens, level) end # @@ -532,8 +539,8 @@ defmodule Philomena.Textile.ParserMarkdown do # (block_textile_element | TOKEN)* eos; # - defp textile(parser, tokens) do - case block_textile_element(parser, tokens) do + defp textile(parser, tokens, level) do + case block_textile_element(parser, tokens, level) do {:ok, tree, r_tokens} -> {:ok, tree, r_tokens} diff --git a/lib/philomena_web/controllers/post/preview_controller.ex b/lib/philomena_web/controllers/post/preview_controller.ex index b5a34e75..dd1b235b 100644 --- a/lib/philomena_web/controllers/post/preview_controller.ex +++ b/lib/philomena_web/controllers/post/preview_controller.ex @@ -10,7 +10,7 @@ defmodule PhilomenaWeb.Post.PreviewController do body = to_string(params["body"]) anonymous = params["anonymous"] == true - post = %Post{user: user, body_md: body, anonymous: anonymous} + post = %Post{user: user, body: body, anonymous: anonymous} rendered = TextRenderer.render_one(post, conn) render(conn, "create.html", layout: false, post: post, body: rendered) diff --git a/lib/philomena_web/text_renderer.ex b/lib/philomena_web/text_renderer.ex index 96152a50..7bb2036a 100644 --- a/lib/philomena_web/text_renderer.ex +++ b/lib/philomena_web/text_renderer.ex @@ -1,6 +1,6 @@ defmodule PhilomenaWeb.TextRenderer do - alias PhilomenaWeb.TextileRenderer alias PhilomenaWeb.MarkdownRenderer + alias PhilomenaWeb.TextileMarkdownRenderer def render_one(item, conn) do hd(render_collection([item], conn)) @@ -11,7 +11,8 @@ defmodule PhilomenaWeb.TextRenderer do if Map.has_key?(item, :body_md) && item.body_md != nil && item.body_md != "" do MarkdownRenderer.render(item.body_md, conn) else - TextileRenderer.render(item.body, conn) + markdown = TextileMarkdownRenderer.render_one(item) + MarkdownRenderer.render(markdown, conn) end end) end From 6aad14b806d7703d22453fde5d690ca51c2fbd90 Mon Sep 17 00:00:00 2001 From: Luna D Date: Wed, 15 Sep 2021 00:14:35 +0200 Subject: [PATCH 17/25] make md renderer not escape --- lib/philomena/textile/parser_markdown.ex | 66 ++++++++++-------------- native/philomena/Cargo.lock | 6 +-- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 62b4a65c..40731ab9 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -40,13 +40,6 @@ defmodule Philomena.Textile.ParserMarkdown do |> Enum.map_join("", fn {_k, v} -> v end) end - # Helper to escape HTML - defp escape(text) do - text - |> HTML.html_escape() - |> HTML.safe_to_string() - end - # Helper to turn a parse tree into a list def partial_flatten(tree) do tree @@ -102,7 +95,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:markup, "\n" <> String.duplicate(open_tag, level)}, tree, {:markup, "\n" <> String.duplicate(close_tag, level - 1)}], r2_tokens} {:ok, tree, r2_tokens, level} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end @@ -114,7 +107,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end @@ -154,19 +147,19 @@ defmodule Philomena.Textile.ParserMarkdown do _ -> case r_tokens do [{forbidden_lookahead, _la} | _] when forbidden_lookahead in [:space, :newline] -> - {:ok, [{:text, escape(open)}], r_tokens} + {:ok, [{:text, open}], r_tokens} _ -> case repeat(callback, put_state(parser, state), r_tokens, false, level) do {:ok, tree, [{^close_token, close}, {^lookahead_not, ln} | r2_tokens]} -> - {:ok, [{:text, escape(open)}, tree, {:text, escape(close)}], + {:ok, [{:text, open}, tree, {:text, close}], [{lookahead_not, ln} | r2_tokens]} {:ok, tree, [{^close_token, _} | r2_tokens]} -> {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end end @@ -256,38 +249,38 @@ defmodule Philomena.Textile.ParserMarkdown do ] do case inline_textile_element(parser, [{:quicktxt, q} | r_tokens], level) do {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(t)}, tree], r2_tokens} + {:ok, [{:text, t}, tree], r2_tokens} _ -> - {:ok, [{:text, escape(t)}], [{:quicktxt, q} | r_tokens]} + {:ok, [{:text, t}], [{:quicktxt, q} | r_tokens]} end end defp inner_inline_textile_element(parser, [{:link_delim, open} | r_tokens], level) do case repeat(&block_textile_element/3, parser, r_tokens, false, level) do {:ok, tree, [{:unbracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> - href = escape(url) + href = url {:ok, [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], r2_tokens} {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end defp inner_inline_textile_element(parser, [{:bracketed_link_open, open} | r_tokens], level) do case repeat(&inline_textile_element/3, parser, r_tokens, false, level) do {:ok, tree, [{:bracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> - href = escape(url) + href = url {:ok, [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], r2_tokens} {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end @@ -301,9 +294,9 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [ {:markup, "[![full]("}, - {:markup, escape(img)}, + {:markup, img}, {:markup, ")]("}, - {:markup, escape(url)}, + {:markup, url}, {:markup, ")"} ], r_tokens} end @@ -315,7 +308,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [ {:markup, "![full]("}, - {:markup, escape(img)}, + {:markup, img}, {:markup, ")"} ], r_tokens} end @@ -331,7 +324,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:markup, "`"}, tree, {:markup, "`"}], r2_tokens} {:ok, tree, r2_tokens} -> - {:ok, [{:text, escape(open)}, tree], r2_tokens} + {:ok, [{:text, open}, tree], r2_tokens} end end end @@ -379,7 +372,7 @@ defmodule Philomena.Textile.ParserMarkdown do # defp inline_textile_element_not_opening_markup(_parser, [{:literal, lit} | r_tokens], level) do - {:ok, [{:markup, escape(Markdown.escape_markdown(lit))},], + {:ok, [{:markup, Markdown.escape_markdown(lit)},], r_tokens} end @@ -389,14 +382,7 @@ defmodule Philomena.Textile.ParserMarkdown do defp inline_textile_element_not_opening_markup(_parser, [{:char, lit} | r_tokens], level) do {binary, r2_tokens} = assemble_binary(:char, <>, r_tokens) - - # A bit of an ugly hack to force the parser to output >> instead of >> - case binary do - ">>" -> - {:ok, [{:text, binary}], r2_tokens} - _ -> - {:ok, [{:text, escape(binary)}], r2_tokens} - end + {:ok, [{:text, binary}], r2_tokens} end defp inline_textile_element_not_opening_markup(_parser, [ @@ -414,12 +400,12 @@ defmodule Philomena.Textile.ParserMarkdown do :del_delim, :sub_delim ] do - {:ok, [{:text, escape(<>)}, {:text, escape(t)}, {:text, escape(<>)}], + {:ok, [{:text, <>}, {:text, t}, {:text, <>}], r_tokens} end defp inline_textile_element_not_opening_markup(_parser, [{:quicktxt, lit} | r_tokens], level) do - {:ok, [{:text, escape(<>)}], r_tokens} + {:ok, [{:text, <>}], r_tokens} end defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens], level) do @@ -427,7 +413,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, tree, [{:bq_cite_open, open} | r2_tokens]} -> case repeat(&block_textile_element/4, parser, r2_tokens, true, level + 1) do {:ok, tree2, [{:bq_close, _} | r3_tokens], level} -> - cite = escape(flatten(tree)) + cite = flatten(tree) {:ok, [ @@ -439,20 +425,20 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, tree2, r3_tokens, level} -> {:ok, [ - {:text, escape(start)}, - {:text, escape(flatten(tree))}, - {:text, escape(open)}, + {:text, start}, + {:text, flatten(tree)}, + {:text, open}, tree2 ], r3_tokens} end _ -> - {:ok, [{:text, escape(start)}], r_tokens} + {:ok, [{:text, start}], r_tokens} end end defp inline_textile_element_not_opening_markup(_parser, [{:bq_cite_open, tok} | r_tokens], level) do - {:ok, [{:text, escape(tok)}], r_tokens} + {:ok, [{:text, tok}], r_tokens} end defp inline_textile_element_not_opening_markup(parser, tokens, level) do @@ -547,7 +533,7 @@ defmodule Philomena.Textile.ParserMarkdown do _ -> case tokens do [{_, string} | r_tokens] -> - {:ok, [{:text, escape(string)}], r_tokens} + {:ok, [{:text, string}], r_tokens} _ -> {:error, "Expected textile"} diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index d38cfcf4..45c701b5 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#a4da46ece62bc99a82cfffb37fa7798d1b1da132" +source = "git+https://github.com/philomena-dev/comrak?branch=main#8160b121f3ccfa5b284c03eedb71add59c3fbb9f" dependencies = [ "clap", "entities", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" dependencies = [ "itoa", "ryu", From 35a227e9382856220fb08a82e863da7e67fe5463 Mon Sep 17 00:00:00 2001 From: Luna D Date: Thu, 16 Sep 2021 21:04:15 +0200 Subject: [PATCH 18/25] use gocamo --- native/philomena/Cargo.lock | 8 +------- native/philomena/Cargo.toml | 2 +- native/philomena/src/camo.rs | 10 ++++------ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 45c701b5..21599c7f 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -43,12 +43,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "base16" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27c3610c36aee21ce8ac510e6224498de4228ad772a171ed65643a24693a5a8" - [[package]] name = "base64" version = "0.13.0" @@ -489,7 +483,7 @@ dependencies = [ name = "philomena" version = "0.3.0" dependencies = [ - "base16", + "base64", "comrak", "jemallocator", "ring", diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index 2fb77ea9..5851c997 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -14,7 +14,7 @@ comrak = { git = "https://github.com/philomena-dev/comrak", branch = "main" } rustler = "0.22" jemallocator = "0.3.2" ring = "0.16" -base16 = "0.2" +base64 = "0.13" url = "2.2" [profile.release] diff --git a/native/philomena/src/camo.rs b/native/philomena/src/camo.rs index cdd4d041..882db6fe 100644 --- a/native/philomena/src/camo.rs +++ b/native/philomena/src/camo.rs @@ -13,16 +13,14 @@ fn untrusted_host(url: Url, camo_host: String, camo_key: String) -> Option Date: Sun, 19 Sep 2021 23:17:50 +0200 Subject: [PATCH 19/25] implement >>image syntax --- lib/philomena/markdown.ex | 4 +- lib/philomena/native.ex | 4 +- lib/philomena_web/markdown_renderer.ex | 93 +++++++++++++++++++++++++- native/philomena/Cargo.lock | 42 ++++++------ native/philomena/src/lib.rs | 9 +-- native/philomena/src/markdown.rs | 21 +++++- 6 files changed, 139 insertions(+), 34 deletions(-) diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index fbff2a48..cc609f94 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -2,8 +2,8 @@ defmodule Philomena.Markdown do @markdown_chars ~r/[\*_\[\]\(\)\^`\%\\~<>#\|]/ # When your NIF is loaded, it will override this function. - def to_html(text), do: Philomena.Native.markdown_to_html(text) - def to_html_unsafe(text), do: Philomena.Native.markdown_to_html_unsafe(text) + def to_html(text, replacements), do: Philomena.Native.markdown_to_html(text, replacements) + def to_html_unsafe(text, replacements), do: Philomena.Native.markdown_to_html_unsafe(text, replacements) def escape_markdown(text) do @markdown_chars diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index 4d9f5ebf..04f700ab 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -2,8 +2,8 @@ defmodule Philomena.Native do use Rustler, otp_app: :philomena # Markdown - def markdown_to_html(_text), do: :erlang.nif_error(:nif_not_loaded) - def markdown_to_html_unsafe(_text), do: :erlang.nif_error(:nif_not_loaded) + def markdown_to_html(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded) + def markdown_to_html_unsafe(_text, _replacements), do: :erlang.nif_error(:nif_not_loaded) # Camo def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex index 72e7376a..4ec85c6f 100644 --- a/lib/philomena_web/markdown_renderer.ex +++ b/lib/philomena_web/markdown_renderer.ex @@ -1,7 +1,96 @@ defmodule PhilomenaWeb.MarkdownRenderer do alias Philomena.Markdown + alias Philomena.Images.Image + alias Philomena.Repo + import Phoenix.HTML + import Phoenix.HTML.Link + import Ecto.Query - def render(text, _conn) do - Markdown.to_html(text) + @image_view Module.concat(["PhilomenaWeb.ImageView"]) + + def render(text, conn) do + images = find_images(text) + representations = render_representations(images, conn) + + Markdown.to_html(text, representations) + end + + defp find_images(text) do + Regex.scan(~r/>>(\d+)([tsp])?/, text, capture: :all_but_first) + |> Enum.map(fn matches -> [Enum.at(matches, 0) |> String.to_integer(), Enum.at(matches, 1) || ""] end) + |> Enum.filter(fn m -> Enum.at(m, 0) < 2_147_483_647 end) + end + + defp load_images(images) do + ids = Enum.map(images, fn m -> Enum.at(m, 0) end) + + Image + |> where([i], i.id in ^ids) + |> preload(tags: :aliases) + |> Repo.all() + |> Map.new(&{&1.id, &1}) + end + + defp link_suffix(image) do + cond do + not is_nil(image.duplicate_id) -> + " (merged)" + + image.hidden_from_users -> + " (deleted)" + + true -> + "" + end + end + + defp render_representations(images, conn) do + loaded_images = load_images(images) + + images + |> Enum.map(fn group -> + img = loaded_images[Enum.at(group, 0)] + text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}" + + rendered = cond do + img != nil -> + case group do + [_id, "p"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :medium, + conn: conn + ) + |> safe_to_string() + [_id, "t"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :small, + conn: conn + ) + |> safe_to_string() + [_id, "s"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :thumb_small, + conn: conn + ) + |> safe_to_string() + [_id, ""] -> + link(">>#{img.id}#{link_suffix(img)}", to: "/images/#{img.id}") + |> safe_to_string() + [_id, suffix] when suffix in ["t", "s", "p"] -> + link(">>#{img.id}#{suffix}#{link_suffix(img)}", to: "/images/#{img.id}") + |> safe_to_string() + [id, suffix] -> # This condition should never trigger, but let's leave it here just in case. + ">>#{id}#{suffix}" + end + true -> + ">>#{text}" + end + + [text, rendered] + end) + |> Map.new(fn [id, html] -> {id, html} end) end end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 21599c7f..ab588a5e 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -87,9 +87,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" [[package]] name = "byte-tools" @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#8160b121f3ccfa5b284c03eedb71add59c3fbb9f" +source = "git+https://github.com/philomena-dev/comrak?branch=main#3ecac2425999185fee33991dec6efbe812d82196" dependencies = [ "clap", "entities", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1866b355d9c878e5e607473cbe3f63282c0b7aad2db1dbebf55076c686918254" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" dependencies = [ "wasm-bindgen", ] @@ -327,9 +327,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.101" +version = "0.2.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" [[package]] name = "line-wrap" @@ -785,9 +785,9 @@ checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" [[package]] name = "unicode-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" @@ -853,9 +853,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -863,9 +863,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -878,9 +878,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d5a6580be83b19dc570a8f9c324251687ab2184e57086f71625feb57ec77c8" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -888,9 +888,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3775a030dc6f5a0afd8a84981a21cc92a781eb429acef9ecce476d0c9113e92" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -901,15 +901,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" [[package]] name = "web-sys" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a84d70d1ec7d2da2d26a5bd78f4bca1b8c3254805363ce743b7a05bc30d195a" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 6cc393f6..ada6eb4b 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,4 +1,5 @@ use jemallocator::Jemalloc; +use rustler::Term; mod camo; mod markdown; @@ -14,13 +15,13 @@ rustler::init! { // Markdown NIF wrappers. #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html(input: String) -> String { - markdown::to_html(input) +fn markdown_to_html<'a>(input: String, reps: Term<'a>) -> String { + markdown::to_html(input, reps) } #[rustler::nif(schedule = "DirtyCpu")] -fn markdown_to_html_unsafe(input: String) -> String { - markdown::to_html_unsafe(input) +fn markdown_to_html_unsafe<'a>(input: String, reps: Term<'a>) -> String { + markdown::to_html_unsafe(input, reps) } // Camo NIF wrappers. diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index af74785a..ff2cf0d2 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -1,5 +1,7 @@ use comrak::ComrakOptions; use crate::camo; +use rustler::{MapIterator, Term}; +use std::collections::HashMap; fn common_options() -> ComrakOptions { let mut options = ComrakOptions::default(); @@ -13,21 +15,34 @@ fn common_options() -> ComrakOptions { options.render.hardbreaks = true; options.render.github_pre_lang = true; - options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); + options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or(String::from(""))); options } -pub fn to_html(input: String) -> String { +fn map_to_hashmap<'a>(map: Term<'a>) -> Option> { + Some(MapIterator::new(map)?.map(|(key, value)| { + let key: String = key.decode().unwrap_or_else(|_| String::from("")); + let value: String = value.decode().unwrap_or_else(|_| String::from("")); + + (key, value) + }).collect()) +} + +pub fn to_html<'a>(input: String, reps: Term<'a>) -> String { let mut options = common_options(); options.render.escape = true; + options.extension.philomena_replacements = map_to_hashmap(reps); + comrak::markdown_to_html(&input, &options) } -pub fn to_html_unsafe(input: String) -> String { +pub fn to_html_unsafe<'a>(input: String, reps: Term<'a>) -> String { let mut options = common_options(); options.render.unsafe_ = true; + options.extension.philomena_replacements = map_to_hashmap(reps); + comrak::markdown_to_html(&input, &options) } From 4e9cde610af68353800a74e9dace6e46b10eeba1 Mon Sep 17 00:00:00 2001 From: Luna D Date: Sun, 19 Sep 2021 23:22:48 +0200 Subject: [PATCH 20/25] mix format --- lib/philomena/markdown.ex | 4 +- lib/philomena/textile/lexer.ex | 6 +- lib/philomena/textile/parser_markdown.ex | 95 +++++++++++++------ .../image/description_controller.ex | 3 +- .../controllers/profile_controller.ex | 11 ++- .../controllers/tag_controller.ex | 3 +- lib/philomena_web/image_loader.ex | 3 +- lib/philomena_web/markdown_renderer.ex | 84 ++++++++-------- lib/philomena_web/textile_renderer.ex | 2 +- lib/philomena_web/views/error_view.ex | 13 +-- native/philomena/src/markdown.rs | 2 +- .../20210912171343_add_markdown_columns.exs | 4 +- 12 files changed, 141 insertions(+), 89 deletions(-) diff --git a/lib/philomena/markdown.ex b/lib/philomena/markdown.ex index cc609f94..a4f3cfa7 100644 --- a/lib/philomena/markdown.ex +++ b/lib/philomena/markdown.ex @@ -3,7 +3,9 @@ defmodule Philomena.Markdown do # When your NIF is loaded, it will override this function. def to_html(text, replacements), do: Philomena.Native.markdown_to_html(text, replacements) - def to_html_unsafe(text, replacements), do: Philomena.Native.markdown_to_html_unsafe(text, replacements) + + def to_html_unsafe(text, replacements), + do: Philomena.Native.markdown_to_html_unsafe(text, replacements) def escape_markdown(text) do @markdown_chars diff --git a/lib/philomena/textile/lexer.ex b/lib/philomena/textile/lexer.ex index f6b2cef7..7fe65347 100644 --- a/lib/philomena/textile/lexer.ex +++ b/lib/philomena/textile/lexer.ex @@ -2,9 +2,9 @@ defmodule Philomena.Textile.Lexer do import NimbleParsec token_list = - Enum.to_list(0x01..0x29) - ++ Enum.to_list(0x2b..0x2f) - ++ ':;<=>?[]\\^`~|' + Enum.to_list(0x01..0x29) ++ + Enum.to_list(0x2B..0x2F) ++ + ':;<=>?[]\\^`~|' space_list = '\f \r\t\u00a0\u1680\u180e\u202f\u205f\u3000' ++ Enum.to_list(0x2000..0x200A) space = utf8_char(space_list) diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 40731ab9..6d4854fa 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -87,21 +87,44 @@ defmodule Philomena.Textile.ParserMarkdown do # # open_token callback* close_token # - defp simple_recursive(:bq_open, :bq_close, open_tag, close_tag, callback, parser, [ - {:bq_open, open} | r_tokens - ], level) do + defp simple_recursive( + :bq_open, + :bq_close, + open_tag, + close_tag, + callback, + parser, + [ + {:bq_open, open} | r_tokens + ], + level + ) do case repeat(callback, parser, r_tokens, true, level) do {:ok, tree, [{:bq_close, _} | r2_tokens], level} -> - {:ok, [{:markup, "\n" <> String.duplicate(open_tag, level)}, tree, {:markup, "\n" <> String.duplicate(close_tag, level - 1)}], r2_tokens} + {:ok, + [ + {:markup, "\n" <> String.duplicate(open_tag, level)}, + tree, + {:markup, "\n" <> String.duplicate(close_tag, level - 1)} + ], r2_tokens} {:ok, tree, r2_tokens, level} -> {:ok, [{:text, open}, tree], r2_tokens} end end - defp simple_recursive(open_token, close_token, open_tag, close_tag, callback, parser, [ - {open_token, open} | r_tokens - ], level) do + defp simple_recursive( + open_token, + close_token, + open_tag, + close_tag, + callback, + parser, + [ + {open_token, open} | r_tokens + ], + level + ) do case repeat(callback, parser, r_tokens, false, level) do {:ok, tree, [{^close_token, _} | r2_tokens]} -> {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} @@ -152,8 +175,7 @@ defmodule Philomena.Textile.ParserMarkdown do _ -> case repeat(callback, put_state(parser, state), r_tokens, false, level) do {:ok, tree, [{^close_token, close}, {^lookahead_not, ln} | r2_tokens]} -> - {:ok, [{:text, open}, tree, {:text, close}], - [{lookahead_not, ln} | r2_tokens]} + {:ok, [{:text, open}, tree, {:text, close}], [{lookahead_not, ln} | r2_tokens]} {:ok, tree, [{^close_token, _} | r2_tokens]} -> {:ok, [{:markup, open_tag}, tree, {:markup, close_tag}], r2_tokens} @@ -261,9 +283,7 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, tree, [{:unbracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> href = url - {:ok, - [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], - r2_tokens} + {:ok, [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], r2_tokens} {:ok, tree, r2_tokens} -> {:ok, [{:text, open}, tree], r2_tokens} @@ -275,19 +295,21 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, tree, [{:bracketed_link_url, <<"\":", url::binary>>} | r2_tokens]} -> href = url - {:ok, - [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], - r2_tokens} + {:ok, [{:markup, "["}, tree, {:markup, "]("}, {:markup, href}, {:markup, ")"}], r2_tokens} {:ok, tree, r2_tokens} -> {:ok, [{:text, open}, tree], r2_tokens} end end - defp inner_inline_textile_element(parser, [ - {token, img}, - {:unbracketed_image_url, <<":", url::binary>>} | r_tokens - ], level) + defp inner_inline_textile_element( + parser, + [ + {token, img}, + {:unbracketed_image_url, <<":", url::binary>>} | r_tokens + ], + level + ) when token in [:unbracketed_image, :bracketed_image] do img = parser.image_transform.(img) @@ -372,8 +394,7 @@ defmodule Philomena.Textile.ParserMarkdown do # defp inline_textile_element_not_opening_markup(_parser, [{:literal, lit} | r_tokens], level) do - {:ok, [{:markup, Markdown.escape_markdown(lit)},], - r_tokens} + {:ok, [{:markup, Markdown.escape_markdown(lit)}], r_tokens} end defp inline_textile_element_not_opening_markup(_parser, [{:space, _} | r_tokens], level) do @@ -385,11 +406,15 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:text, binary}], r2_tokens} end - defp inline_textile_element_not_opening_markup(_parser, [ - {:quicktxt, q1}, - {token, t}, - {:quicktxt, q2} | r_tokens - ], level) + defp inline_textile_element_not_opening_markup( + _parser, + [ + {:quicktxt, q1}, + {token, t}, + {:quicktxt, q2} | r_tokens + ], + level + ) when token in [ :b_delim, :i_delim, @@ -400,15 +425,18 @@ defmodule Philomena.Textile.ParserMarkdown do :del_delim, :sub_delim ] do - {:ok, [{:text, <>}, {:text, t}, {:text, <>}], - r_tokens} + {:ok, [{:text, <>}, {:text, t}, {:text, <>}], r_tokens} end defp inline_textile_element_not_opening_markup(_parser, [{:quicktxt, lit} | r_tokens], level) do {:ok, [{:text, <>}], r_tokens} end - defp inline_textile_element_not_opening_markup(parser, [{:bq_cite_start, start} | r_tokens], level) do + defp inline_textile_element_not_opening_markup( + parser, + [{:bq_cite_start, start} | r_tokens], + level + ) do case repeat(&bq_cite_text/3, parser, r_tokens, false, level) do {:ok, tree, [{:bq_cite_open, open} | r2_tokens]} -> case repeat(&block_textile_element/4, parser, r2_tokens, true, level + 1) do @@ -437,7 +465,11 @@ defmodule Philomena.Textile.ParserMarkdown do end end - defp inline_textile_element_not_opening_markup(_parser, [{:bq_cite_open, tok} | r_tokens], level) do + defp inline_textile_element_not_opening_markup( + _parser, + [{:bq_cite_open, tok} | r_tokens], + level + ) do {:ok, [{:text, tok}], r_tokens} end @@ -495,7 +527,8 @@ defmodule Philomena.Textile.ParserMarkdown do # double_newline | newline | inline_textile_element; # - defp block_textile_element(_parser, [{:double_newline, _} | r_tokens], bq, level) when bq == true do + defp block_textile_element(_parser, [{:double_newline, _} | r_tokens], bq, level) + when bq == true do one = "\n" <> String.duplicate("> ", level) {:ok, [{:markup, String.duplicate(one, 2)}], r_tokens} end diff --git a/lib/philomena_web/controllers/image/description_controller.ex b/lib/philomena_web/controllers/image/description_controller.ex index 7e508567..55c8191a 100644 --- a/lib/philomena_web/controllers/image/description_controller.ex +++ b/lib/philomena_web/controllers/image/description_controller.ex @@ -34,7 +34,8 @@ defmodule PhilomenaWeb.Image.DescriptionController do Images.reindex_image(image) - body = TextRenderer.render_one(%{body: image.description, body_md: image.description_md}, conn) + body = + TextRenderer.render_one(%{body: image.description, body_md: image.description_md}, conn) conn |> put_view(PhilomenaWeb.ImageView) diff --git a/lib/philomena_web/controllers/profile_controller.ex b/lib/philomena_web/controllers/profile_controller.ex index b0cb6a2d..9d35f491 100644 --- a/lib/philomena_web/controllers/profile_controller.ex +++ b/lib/philomena_web/controllers/profile_controller.ex @@ -134,9 +134,11 @@ defmodule PhilomenaWeb.ProfileController do |> TextRenderer.render_collection(conn) |> Enum.zip(recent_comments) - about_me = TextRenderer.render_one(%{body_md: user.description_md, body: user.description || ""}, conn) + about_me = + TextRenderer.render_one(%{body_md: user.description_md, body: user.description || ""}, conn) - scratchpad = TextRenderer.render_one(%{body_md: user.scratchpad_md, body: user.scratchpad || ""}, conn) + scratchpad = + TextRenderer.render_one(%{body_md: user.scratchpad_md, body: user.scratchpad || ""}, conn) commission_information = commission_info(user.commission, conn) @@ -214,8 +216,9 @@ defmodule PhilomenaWeb.ProfileController do defp map_fetch(nil, _field_name), do: nil defp map_fetch(map, field_name), do: Map.get(map, field_name) - defp commission_info(%{information: info, information_md: info_md}, conn) when info not in [nil, ""], - do: TextRenderer.render_one(%{body: info, body_md: info_md}, conn) + defp commission_info(%{information: info, information_md: info_md}, conn) + when info not in [nil, ""], + do: TextRenderer.render_one(%{body: info, body_md: info_md}, conn) defp commission_info(_commission, _conn), do: "" diff --git a/lib/philomena_web/controllers/tag_controller.ex b/lib/philomena_web/controllers/tag_controller.ex index ee4c6f3b..5713c61b 100644 --- a/lib/philomena_web/controllers/tag_controller.ex +++ b/lib/philomena_web/controllers/tag_controller.ex @@ -61,7 +61,8 @@ defmodule PhilomenaWeb.TagController do interactions = Interactions.user_interactions(images, user) - body = TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) + body = + TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) dnp_bodies = TextRenderer.render_collection( diff --git a/lib/philomena_web/image_loader.ex b/lib/philomena_web/image_loader.ex index 914404e6..86b0c3b9 100644 --- a/lib/philomena_web/image_loader.ex +++ b/lib/philomena_web/image_loader.ex @@ -139,7 +139,8 @@ defmodule PhilomenaWeb.ImageLoader do dnp_entries = Enum.zip(dnp_bodies, tag.dnp_entries) - description = TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) + description = + TextRenderer.render_one(%{body_md: tag.description_md, body: tag.description || ""}, conn) [{tag, description, dnp_entries}] end diff --git a/lib/philomena_web/markdown_renderer.ex b/lib/philomena_web/markdown_renderer.ex index 4ec85c6f..bf4259db 100644 --- a/lib/philomena_web/markdown_renderer.ex +++ b/lib/philomena_web/markdown_renderer.ex @@ -17,7 +17,9 @@ defmodule PhilomenaWeb.MarkdownRenderer do defp find_images(text) do Regex.scan(~r/>>(\d+)([tsp])?/, text, capture: :all_but_first) - |> Enum.map(fn matches -> [Enum.at(matches, 0) |> String.to_integer(), Enum.at(matches, 1) || ""] end) + |> Enum.map(fn matches -> + [Enum.at(matches, 0) |> String.to_integer(), Enum.at(matches, 1) || ""] + end) |> Enum.filter(fn m -> Enum.at(m, 0) < 2_147_483_647 end) end @@ -52,42 +54,50 @@ defmodule PhilomenaWeb.MarkdownRenderer do img = loaded_images[Enum.at(group, 0)] text = "#{Enum.at(group, 0)}#{Enum.at(group, 1)}" - rendered = cond do - img != nil -> - case group do - [_id, "p"] when not img.hidden_from_users -> - Phoenix.View.render(@image_view, "_image_target.html", - image: img, - size: :medium, - conn: conn - ) - |> safe_to_string() - [_id, "t"] when not img.hidden_from_users -> - Phoenix.View.render(@image_view, "_image_target.html", - image: img, - size: :small, - conn: conn - ) - |> safe_to_string() - [_id, "s"] when not img.hidden_from_users -> - Phoenix.View.render(@image_view, "_image_target.html", - image: img, - size: :thumb_small, - conn: conn - ) - |> safe_to_string() - [_id, ""] -> - link(">>#{img.id}#{link_suffix(img)}", to: "/images/#{img.id}") - |> safe_to_string() - [_id, suffix] when suffix in ["t", "s", "p"] -> - link(">>#{img.id}#{suffix}#{link_suffix(img)}", to: "/images/#{img.id}") - |> safe_to_string() - [id, suffix] -> # This condition should never trigger, but let's leave it here just in case. - ">>#{id}#{suffix}" - end - true -> - ">>#{text}" - end + rendered = + cond do + img != nil -> + case group do + [_id, "p"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :medium, + conn: conn + ) + |> safe_to_string() + + [_id, "t"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :small, + conn: conn + ) + |> safe_to_string() + + [_id, "s"] when not img.hidden_from_users -> + Phoenix.View.render(@image_view, "_image_target.html", + image: img, + size: :thumb_small, + conn: conn + ) + |> safe_to_string() + + [_id, ""] -> + link(">>#{img.id}#{link_suffix(img)}", to: "/images/#{img.id}") + |> safe_to_string() + + [_id, suffix] when suffix in ["t", "s", "p"] -> + link(">>#{img.id}#{suffix}#{link_suffix(img)}", to: "/images/#{img.id}") + |> safe_to_string() + + # This condition should never trigger, but let's leave it here just in case. + [id, suffix] -> + ">>#{id}#{suffix}" + end + + true -> + ">>#{text}" + end [text, rendered] end) diff --git a/lib/philomena_web/textile_renderer.ex b/lib/philomena_web/textile_renderer.ex index 50870dfd..b3e465c6 100644 --- a/lib/philomena_web/textile_renderer.ex +++ b/lib/philomena_web/textile_renderer.ex @@ -12,7 +12,7 @@ defmodule PhilomenaWeb.TextileRenderer do def render(text, conn) do opts = %{image_transform: &Camo.Image.image_url/1} parsed = Parser.parse(opts, text) - + images = parsed |> Enum.flat_map(fn diff --git a/lib/philomena_web/views/error_view.ex b/lib/philomena_web/views/error_view.ex index 0e7cc087..1439cf87 100644 --- a/lib/philomena_web/views/error_view.ex +++ b/lib/philomena_web/views/error_view.ex @@ -1,17 +1,18 @@ defmodule PhilomenaWeb.ErrorView do use PhilomenaWeb, :view - import PhilomenaWeb.LayoutView, only: [ - stylesheet_path: 2, - dark_stylesheet_path: 1, - viewport_meta_tag: 1 - ] + import PhilomenaWeb.LayoutView, + only: [ + stylesheet_path: 2, + dark_stylesheet_path: 1, + viewport_meta_tag: 1 + ] @codes %{ 400 => {"Bad Request", "Couldn't process your request!"}, 403 => {"Forbidden", "Not allowed to access this page (are your cookies enabled?)"}, 404 => {"Not Found", "Couldn't find what you were looking for!"}, - 500 => {"Internal Error", "Couldn't process your request!"}, + 500 => {"Internal Error", "Couldn't process your request!"} } # By default, Phoenix returns the status message from diff --git a/native/philomena/src/markdown.rs b/native/philomena/src/markdown.rs index ff2cf0d2..b6345cba 100644 --- a/native/philomena/src/markdown.rs +++ b/native/philomena/src/markdown.rs @@ -15,7 +15,7 @@ fn common_options() -> ComrakOptions { options.render.hardbreaks = true; options.render.github_pre_lang = true; - options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or(String::from(""))); + options.extension.camoifier = Some(|s| camo::image_url(s).unwrap_or_else(|| String::from(""))); options } diff --git a/priv/repo/migrations/20210912171343_add_markdown_columns.exs b/priv/repo/migrations/20210912171343_add_markdown_columns.exs index 21267871..45961854 100644 --- a/priv/repo/migrations/20210912171343_add_markdown_columns.exs +++ b/priv/repo/migrations/20210912171343_add_markdown_columns.exs @@ -5,7 +5,7 @@ defmodule Philomena.Repo.Migrations.AddMarkdownColumns do alter table("comments") do add :body_md, :varchar, default: nil end - + alter table("messages") do add :body_md, :varchar, default: nil end @@ -75,7 +75,7 @@ defmodule Philomena.Repo.Migrations.AddMarkdownColumns do alter table("comments") do remove :body_md end - + alter table("messages") do remove :body_md end From 49388920114740da26f7ac89aa5fc30349e43063 Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 20 Sep 2021 20:24:44 +0200 Subject: [PATCH 21/25] improve the looks of blockquotes when nested a lot --- assets/css/common/_base.scss | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/assets/css/common/_base.scss b/assets/css/common/_base.scss index a59f2309..3e1fe060 100644 --- a/assets/css/common/_base.scss +++ b/assets/css/common/_base.scss @@ -210,6 +210,20 @@ blockquote { background-color: inherit; } +// Prevent blockquote from gaining far too much indentation and breaking. +blockquote blockquote blockquote blockquote blockquote blockquote { + margin: 1em 0; + padding: 1em 2px; +} + +// Horizontal space is at a high premium on mobile. +@media (max-width: $min_px_width_for_desktop_layout) { + blockquote { + margin: 1em 4px; + padding: 1em 4px; + } +} + .spoiler { background-color: $admin_links_hover_color; color: $admin_links_hover_color; From 79ec9426ca629aea24370f85303512c2f620a00a Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 20 Sep 2021 22:42:04 +0200 Subject: [PATCH 22/25] update comrak --- native/philomena/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index ab588a5e..d313943b 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "comrak" version = "0.12.1" -source = "git+https://github.com/philomena-dev/comrak?branch=main#3ecac2425999185fee33991dec6efbe812d82196" +source = "git+https://github.com/philomena-dev/comrak?branch=main#3e2120b6b5d5d995b787dad27699f5bda4bc6112" dependencies = [ "clap", "entities", From a32f01ae1a93a9481eb5517872c20c207793dc1a Mon Sep 17 00:00:00 2001 From: Luna D Date: Mon, 20 Sep 2021 23:54:06 +0200 Subject: [PATCH 23/25] fix thing --- lib/philomena/textile/parser_markdown.ex | 73 ++++++++++++++++-------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 6d4854fa..70a2f5ba 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -40,6 +40,14 @@ defmodule Philomena.Textile.ParserMarkdown do |> Enum.map_join("", fn {_k, v} -> v end) end + def flatten_unquote(tree) do + tree + |> List.flatten() + |> Enum.map_join("", fn {_k, v} -> + Regex.replace(~r/\n(> )/, v, "\n") + end) + end + # Helper to turn a parse tree into a list def partial_flatten(tree) do tree @@ -473,9 +481,39 @@ defmodule Philomena.Textile.ParserMarkdown do {:ok, [{:text, tok}], r_tokens} end + defp inline_textile_element_not_opening_markup( + parser, + [{:bq_open, start} | r_tokens], + level + ) do + case repeat(&block_textile_element/4, parser, r_tokens, true, level + 1) do + {:ok, tree, [{:bq_close, _} | r2_tokens], level} -> + {:ok, + [ + {:markup, "\n" <> String.duplicate("> ", level)}, + tree, + {:markup, "\n" <> String.duplicate("> ", level - 1)} + ], r2_tokens} + + {:ok, tree, r2_tokens, level} -> + {:ok, + [ + {:text, start}, + {:text, flatten_unquote(tree)} + ], r2_tokens} + end + end + + defp inline_textile_element_not_opening_markup( + _parser, + [{:bq_open, tok} | r_tokens], + level + ) do + {:ok, [{:text, tok}], r_tokens} + end + defp inline_textile_element_not_opening_markup(parser, tokens, level) do [ - {:bq_open, :bq_close, "> ", "> "}, {:spoiler_open, :spoiler_close, "||", "||"}, {:bracketed_b_open, :bracketed_b_close, "**", "**"}, {:bracketed_i_open, :bracketed_i_close, "_", "_"}, @@ -488,29 +526,16 @@ defmodule Philomena.Textile.ParserMarkdown do {:bracketed_sub_open, :bracketed_sub_close, "%", "%"} ] |> Enum.find_value(fn {open_token, close_token, open_tag, close_tag} -> - if open_token == :bq_open do - simple_recursive( - open_token, - close_token, - open_tag, - close_tag, - &block_textile_element/4, - parser, - tokens, - level + 1 - ) - else - simple_recursive( - open_token, - close_token, - open_tag, - close_tag, - &block_textile_element/3, - parser, - tokens, - level - ) - end + simple_recursive( + open_token, + close_token, + open_tag, + close_tag, + &block_textile_element/3, + parser, + tokens, + level + ) |> case do {:ok, tree, r_tokens} -> {:ok, tree, r_tokens} From d97f348a5012f697fc0ffbd431c52eab33f1c91e Mon Sep 17 00:00:00 2001 From: Luna D Date: Tue, 21 Sep 2021 00:55:15 +0200 Subject: [PATCH 24/25] mix format complained about this file so i guess? --- lib/philomena/textile/parser_markdown.ex | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/philomena/textile/parser_markdown.ex b/lib/philomena/textile/parser_markdown.ex index 70a2f5ba..a6e1e80e 100644 --- a/lib/philomena/textile/parser_markdown.ex +++ b/lib/philomena/textile/parser_markdown.ex @@ -43,7 +43,7 @@ defmodule Philomena.Textile.ParserMarkdown do def flatten_unquote(tree) do tree |> List.flatten() - |> Enum.map_join("", fn {_k, v} -> + |> Enum.map_join("", fn {_k, v} -> Regex.replace(~r/\n(> )/, v, "\n") end) end @@ -482,25 +482,25 @@ defmodule Philomena.Textile.ParserMarkdown do end defp inline_textile_element_not_opening_markup( - parser, - [{:bq_open, start} | r_tokens], - level - ) do + parser, + [{:bq_open, start} | r_tokens], + level + ) do case repeat(&block_textile_element/4, parser, r_tokens, true, level + 1) do {:ok, tree, [{:bq_close, _} | r2_tokens], level} -> {:ok, - [ - {:markup, "\n" <> String.duplicate("> ", level)}, - tree, - {:markup, "\n" <> String.duplicate("> ", level - 1)} - ], r2_tokens} + [ + {:markup, "\n" <> String.duplicate("> ", level)}, + tree, + {:markup, "\n" <> String.duplicate("> ", level - 1)} + ], r2_tokens} {:ok, tree, r2_tokens, level} -> {:ok, - [ - {:text, start}, - {:text, flatten_unquote(tree)} - ], r2_tokens} + [ + {:text, start}, + {:text, flatten_unquote(tree)} + ], r2_tokens} end end From 2cb49a8df97a2d3234ffa0cea4793380884d4015 Mon Sep 17 00:00:00 2001 From: Luna D Date: Tue, 21 Sep 2021 00:58:15 +0200 Subject: [PATCH 25/25] unnecessary env var --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4ac96e4b..266754d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,5 +82,3 @@ services: - app ports: - '8080:80' - environment: - - RUST_LOG=trace