diff --git a/lib/textile/parser.ex b/lib/textile/parser.ex
index 67c59b86..22c5457c 100644
--- a/lib/textile/parser.ex
+++ b/lib/textile/parser.ex
@@ -13,8 +13,9 @@ defmodule Textile.Parser do
def parse(%Parser{} = parser, input) do
with {:ok, tokens, _1, _2, _3, _4} <- Lexer.lex(input),
- tokens <- TokenCoalescer.coalesce(tokens),
- {:ok, tree, []} <- textile_top(parser, tokens)
+ tokens <- TokenCoalescer.coalesce_lex(tokens),
+ {:ok, tree, []} <- textile_top(parser, tokens),
+ tree <- TokenCoalescer.coalesce_parse(tree)
do
tree
else
@@ -38,7 +39,7 @@ defmodule Textile.Parser do
false <- tree == [],
{:ok, next_tree, r2_tokens} <- textile_top(parser, r_tokens)
do
- {:ok, tree ++ next_tree, r2_tokens}
+ {:ok, [tree, next_tree], r2_tokens}
else
_ ->
[{_token, string} | r_tokens] = tokens
@@ -55,7 +56,7 @@ defmodule Textile.Parser do
defp well_formed_including_paragraphs(_parser, []), do: {:ok, [], []}
defp well_formed_including_paragraphs(parser, [{:double_newline, _nl} | r_tokens]) do
with {:ok, tree, r2_tokens} <- well_formed_including_paragraphs(parser, r_tokens) do
- {:ok, [markup: "
"] ++ tree, r2_tokens}
+ {:ok, [{:markup, "
"}, tree], r2_tokens}
else
_ ->
{:ok, [], r_tokens}
@@ -66,7 +67,7 @@ defmodule Textile.Parser do
with {:ok, tree, r_tokens} <- markup(parser, tokens),
{:ok, next_tree, r2_tokens} <- well_formed_including_paragraphs(parser, r_tokens)
do
- {:ok, tree ++ next_tree, r2_tokens}
+ {:ok, [tree, next_tree], r2_tokens}
else
_ ->
{:ok, [], tokens}
@@ -80,7 +81,7 @@ defmodule Textile.Parser do
case markup(parser, tokens) do
{:ok, tree, r_tokens} ->
{:ok, next_tree, r2_tokens} = well_formed(parser, r_tokens)
- {:ok, tree ++ next_tree, r2_tokens}
+ {:ok, [tree, next_tree], r2_tokens}
_ ->
{:ok, [], tokens}
@@ -124,7 +125,7 @@ defmodule Textile.Parser do
#
defp blockquote(parser, [{:blockquote_open_cite, author} | r_tokens]) do
with {:ok, tree, [{:blockquote_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do
- {:ok, [markup: ~s|
|] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s|
|}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(~s|[bq="#{author}"]|)], r_tokens} @@ -133,7 +134,7 @@ defmodule Textile.Parser do defp blockquote(parser, [{:blockquote_open, open} | r_tokens]) do with {:ok, tree, [{:blockquote_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s|
|] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s|
|}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(open)], r_tokens} @@ -150,7 +151,7 @@ defmodule Textile.Parser do # defp spoiler(parser, [{:spoiler_open, open} | r_tokens]) do with {:ok, tree, [{:spoiler_close, _close} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s||] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s||}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(open)], r_tokens} @@ -167,7 +168,7 @@ defmodule Textile.Parser do # defp link(parser, [{:link_start, start} | r_tokens]) do with {:ok, tree, [{:link_end, _end}, {:link_url, url} | r2_tokens]} <- well_formed_including_paragraphs(parser, r_tokens) do - {:ok, [markup: ~s||] ++ tree ++ [markup: ~s||], r2_tokens} + {:ok, [{:markup, ~s||}, tree, {:markup, ~s||}], r2_tokens} else _ -> {:ok, [text: escape_nl2br(start)], r_tokens} diff --git a/lib/textile/parser_helpers.ex b/lib/textile/parser_helpers.ex index 634b049f..39404cf5 100644 --- a/lib/textile/parser_helpers.ex +++ b/lib/textile/parser_helpers.ex @@ -5,7 +5,7 @@ defmodule Textile.ParserHelpers do quote do defp unquote(name)(parser, [{unquote(open_token), open} | r_tokens]) do with {:ok, tree, [{unquote(close_token), _close} | r2_tokens]} <- well_formed(parser, r_tokens) do - {:ok, [markup: unquote(open_tag)] ++ tree ++ [markup: unquote(close_tag)], r2_tokens} + {:ok, [{:markup, unquote(open_tag)}, tree, {:markup, unquote(close_tag)}], r2_tokens} else _ -> {:ok, [text: escape_html(open)], r_tokens} @@ -14,7 +14,7 @@ defmodule Textile.ParserHelpers do defp unquote(name)(parser, [{unquote(:"b_#{open_token}"), open} | r_tokens]) do with {:ok, tree, [{unquote(:"b_#{close_token}"), _close} | r2_tokens]} <- well_formed(parser, r_tokens) do - {:ok, [markup: unquote(open_tag)] ++ tree ++ [markup: unquote(close_tag)], r2_tokens} + {:ok, [{:markup, unquote(open_tag)}, tree, {:markup, unquote(close_tag)}], r2_tokens} else _ -> {:ok, [text: escape_html(open)], r_tokens} diff --git a/lib/textile/token_coalescer.ex b/lib/textile/token_coalescer.ex index 526545b1..65f02a96 100644 --- a/lib/textile/token_coalescer.ex +++ b/lib/textile/token_coalescer.ex @@ -1,7 +1,7 @@ defmodule Textile.TokenCoalescer do # The lexer, as a practical concern, does not coalesce runs of # character tokens. This fixes that. - def coalesce(tokens) do + def coalesce_lex(tokens) do tokens |> Enum.chunk_by(&is_number(&1)) |> Enum.flat_map(fn @@ -12,4 +12,20 @@ defmodule Textile.TokenCoalescer do t end) end + + def coalesce_parse(tokens) do + tokens + |> List.flatten() + |> Enum.chunk_by(fn {k, _v} -> k == :text end) + |> Enum.flat_map(fn t -> + [{type, _v} | _rest] = t + + value = + t + |> Enum.map(fn {_k, v} -> v end) + |> Enum.join("") + + [{type, value}] + end) + end end \ No newline at end of file