diff --git a/lib/textile/parser.ex b/lib/textile/parser.ex index 67c59b86..22c5457c 100644 --- a/lib/textile/parser.ex +++ b/lib/textile/parser.ex @@ -13,8 +13,9 @@ defmodule Textile.Parser do def parse(%Parser{} = parser, input) do with {:ok, tokens, _1, _2, _3, _4} <- Lexer.lex(input), - tokens <- TokenCoalescer.coalesce(tokens), - {:ok, tree, []} <- textile_top(parser, tokens) + tokens <- TokenCoalescer.coalesce_lex(tokens), + {:ok, tree, []} <- textile_top(parser, tokens), + tree <- TokenCoalescer.coalesce_parse(tree) do tree else @@ -38,7 +39,7 @@ defmodule Textile.Parser do false <- tree == [], {:ok, next_tree, r2_tokens} <- textile_top(parser, r_tokens) do - {:ok, tree ++ next_tree, r2_tokens} + {:ok, [tree, next_tree], r2_tokens} else _ -> [{_token, string} | r_tokens] = tokens @@ -55,7 +56,7 @@ defmodule Textile.Parser do defp well_formed_including_paragraphs(_parser, []), do: {:ok, [], []} defp well_formed_including_paragraphs(parser, [{:double_newline, _nl} | r_tokens]) do with {:ok, tree, r2_tokens} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: "

"] ++ tree, r2_tokens} + {:ok, [{:markup, "

"}, tree], r2_tokens} else _ -> {:ok, [], r_tokens} @@ -66,7 +67,7 @@ defmodule Textile.Parser do with {:ok, tree, r_tokens} <- markup(parser, tokens), {:ok, next_tree, r2_tokens} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, tree ++ next_tree, r2_tokens} + {:ok, [tree, next_tree], r2_tokens} else _ -> {:ok, [], tokens} @@ -80,7 +81,7 @@ defmodule Textile.Parser do case markup(parser, tokens) do {:ok, tree, r_tokens} -> {:ok, next_tree, r2_tokens} = well_formed(parser, r_tokens) - {:ok, tree ++ next_tree, r2_tokens} + {:ok, [tree, next_tree], r2_tokens} _ -> {:ok, [], tokens} @@ -124,7 +125,7 @@ defmodule Textile.Parser do # defp blockquote(parser, [{:blockquote_open_cite, author} | r_tokens]) do with {:ok, tree, [{:blockquote_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s|
|] ++ tree ++ [markup: ~s|
|], r2_tokens} + {:ok, [{:markup, ~s|
|}, tree, {:markup, ~s|
|}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(~s|[bq="#{author}"]|)], r_tokens} @@ -133,7 +134,7 @@ defmodule Textile.Parser do defp blockquote(parser, [{:blockquote_open, open} | r_tokens]) do with {:ok, tree, [{:blockquote_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s|
|] ++ tree ++ [markup: ~s|
|], r2_tokens} + {:ok, [{:markup, ~s|
|}, tree, {:markup, ~s|
|}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(open)], r_tokens} @@ -150,7 +151,7 @@ defmodule Textile.Parser do # defp spoiler(parser, [{:spoiler_open, open} | r_tokens]) do with {:ok, tree, [{:spoiler_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s||] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s||}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(open)], r_tokens} @@ -167,7 +168,7 @@ defmodule Textile.Parser do # defp link(parser, [{:link_start, start} | r_tokens]) do with {:ok, tree, [{:link_end, _end}, {:link_url, url} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s||] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s||}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(start)], r_tokens} diff --git a/lib/textile/parser_helpers.ex b/lib/textile/parser_helpers.ex index 634b049f..39404cf5 100644 --- a/lib/textile/parser_helpers.ex +++ b/lib/textile/parser_helpers.ex @@ -5,7 +5,7 @@ defmodule Textile.ParserHelpers do quote do defp unquote(name)(parser, [{unquote(open_token), open} | r_tokens]) do with {:ok, tree, [{unquote(close_token), _close} | r2_tokens]} <- well_formed(parser, r_tokens) do - {:ok, [markup: unquote(open_tag)] ++ tree ++ [markup: unquote(close_tag)], r2_tokens} + {:ok, [{:markup, unquote(open_tag)}, tree, {:markup, unquote(close_tag)}], r2_tokens} else _ -> {:ok, [text: escape_html(open)], r_tokens} @@ -14,7 +14,7 @@ defmodule Textile.ParserHelpers do defp unquote(name)(parser, [{unquote(:"b_#{open_token}"), open} | r_tokens]) do with {:ok, tree, [{unquote(:"b_#{close_token}"), _close} | r2_tokens]} <- well_formed(parser, r_tokens) do - {:ok, [markup: unquote(open_tag)] ++ tree ++ [markup: unquote(close_tag)], r2_tokens} + {:ok, [{:markup, unquote(open_tag)}, tree, {:markup, unquote(close_tag)}], r2_tokens} else _ -> {:ok, [text: escape_html(open)], r_tokens} diff --git a/lib/textile/token_coalescer.ex b/lib/textile/token_coalescer.ex index 526545b1..65f02a96 100644 --- a/lib/textile/token_coalescer.ex +++ b/lib/textile/token_coalescer.ex @@ -1,7 +1,7 @@ defmodule Textile.TokenCoalescer do # The lexer, as a practical concern, does not coalesce runs of # character tokens. This fixes that. - def coalesce(tokens) do + def coalesce_lex(tokens) do tokens |> Enum.chunk_by(&is_number(&1)) |> Enum.flat_map(fn @@ -12,4 +12,20 @@ defmodule Textile.TokenCoalescer do t end) end + + def coalesce_parse(tokens) do + tokens + |> List.flatten() + |> Enum.chunk_by(fn {k, _v} -> k == :text end) + |> Enum.flat_map(fn t -> + [{type, _v} | _rest] = t + + value = + t + |> Enum.map(fn {_k, v} -> v end) + |> Enum.join("") + + [{type, value}] + end) + end end \ No newline at end of file