philomena/lib/search/parser.ex

defmodule Search.Parser do
  alias Search.{
    BoolParser,
    DateParser,
    FloatParser,
    IntParser,
    IpParser,
    Lexer,
    LiteralParser,
    Parser,
    TermRangeParser
  }

  defstruct [
    :default_field,
    bool_fields: [],
    date_fields: [],
    float_fields: [],
    int_fields: [],
    ip_fields: [],
    literal_fields: [],
    ngram_fields: [],
    custom_fields: [],
    transforms: %{},
    aliases: %{},
    __fields__: %{},
    __data__: nil
  ]

  def parser(options) do
    parser = struct(Parser, options)
    fields =
      Enum.map(parser.bool_fields, fn f -> {f, BoolParser} end) ++
      Enum.map(parser.date_fields, fn f -> {f, DateParser} end) ++
      Enum.map(parser.float_fields, fn f -> {f, FloatParser} end) ++
      Enum.map(parser.int_fields, fn f -> {f, IntParser} end) ++
      Enum.map(parser.ip_fields, fn f -> {f, IpParser} end) ++
      Enum.map(parser.literal_fields, fn f -> {f, LiteralParser} end) ++
      Enum.map(parser.ngram_fields, fn f -> {f, NgramParser} end) ++
      Enum.map(parser.custom_fields, fn f -> {f, :custom_field} end)

    %{parser | __fields__: Map.new(fields)}
  end

  def parse(parser, input, context \\ nil)

  # Empty search should emit a match_none.
  def parse(_parser, "", _context) do
    {:ok, %{match_none: %{}}}
  end

  def parse(%Parser{} = parser, input, context) do
    parser = %{parser | __data__: context}

    with {:ok, tokens, _1, _2, _3, _4} <- Lexer.lex(input),
         {:ok, {tree, []}} <- search_top(parser, tokens)
    do
      {:ok, tree}
    else
      {:ok, {_tree, tokens}} ->
        {:error, "junk at end of expression: " <> debug_tokens(tokens)}

      {:error, msg, start_pos, _1, _2, _3} ->
        {:error, msg <> ", starting at: " <> start_pos}

      {:error, msg} ->
        {:error, msg}

      err ->
        err
        #{:error, "unknown parsing error"}
    end
  end

  defp debug_tokens(tokens) do
    tokens
    |> Enum.map(fn {_k, v} -> v end)
    |> Enum.join("")
  end

  #
  # Predictive LL(1) RD parser for search grammar
  #

  defp search_top(parser, tokens), do: search_or(parser, tokens)

  defp search_or(parser, tokens) do
    with {:ok, {left, [{:or, _} | r_tokens]}} <- search_and(parser, tokens),
         {:ok, {right, rest}} <- search_or(parser, r_tokens)
    do
      {:ok, {flatten_disjunction_child(left, right), rest}}
    else
      value ->
        value
    end
  end

  defp search_and(parser, tokens) do
    with {:ok, {left, [{:and, _} | r_tokens]}} <- search_boost(parser, tokens),
         {:ok, {right, rest}} <- search_and(parser, r_tokens)
    do
      {:ok, {flatten_conjunction_child(left, right), rest}}
    else
      value ->
        value
    end
  end

  defp search_boost(parser, tokens) do
    case search_not(parser, tokens) do
      {:ok, {child, [{:boost, value} | r_tokens]}} ->
        {:ok, {%{function_score: %{query: child, boost_factor: value}}, r_tokens}}

      value ->
        value
    end
  end

  defp search_not(parser, [{:not, _} | rest]) do
    case search_group(parser, rest) do
      {:ok, {child, r_tokens}} ->
        {:ok, {flatten_negation_child(child), r_tokens}}

      value ->
        value
    end
  end

  defp search_not(parser, tokens), do: search_group(parser, tokens)

  defp search_group(parser, [{:lparen, _} | rest]) do
    case search_top(parser, rest) do
      {:ok, {child, [{:rparen, _} | r_tokens]}} ->
        {:ok, {child, r_tokens}}

      {:ok, {_child, _tokens}} ->
        {:error, "Imbalanced parentheses."}

      value ->
        value
    end
  end

  defp search_group(_parser, [{:rparen, _} | _rest]) do
    {:error, "Imbalanced parentheses."}
  end

  defp search_group(parser, tokens), do: search_field(parser, tokens)

  defp search_field(parser, [{:term, value} | r_tokens]) do
    tokens = TermRangeParser.parse(value, parser.__fields__, parser.default_field)

    case field_top(parser, tokens) do
      {:ok, {child, []}} ->
        {:ok, {child, r_tokens}}

      err ->
        err
    end
  end

  defp search_field(_parser, _tokens), do:
    {:error, "Expected a term."}

  #
  # Predictive LL(k) RD parser for search terms in parent grammar
  #

  defp field_top(parser, tokens), do: field_term(parser, tokens)

  defp field_term(parser, [custom_field: field_name, range: :eq, value: value]) do
    case parser.transforms[field_name].(parser.__data__, String.trim(value)) do
      {:ok, child} ->
        {:ok, {child, []}}

      err ->
        err
    end
  end

  defp field_term(parser, [{field_parser, field_name}, {:range, range}, {:value, value}]) do
    # N.B.: field_parser is an atom
    case field_parser.parse(value) do
      {:ok, extra_tokens, _1, _2, _3, _4} ->
        field_type(parser, [{field_parser, field_name}, {:range, range}] ++ extra_tokens)

      err ->
        err
    end   
  end

  # Types which do not support ranges

  defp field_type(parser, [{LiteralParser, field_name}, range: :eq, literal: value]),
    do: {:ok, {%{term: %{field(parser, field_name) => normalize_value(parser, value)}}, []}}

  defp field_type(parser, [{LiteralParser, field_name}, range: :eq, literal: value, fuzz: fuzz]),
    do: {:ok, {%{fuzzy: %{field(parser, field_name) => %{value: normalize_value(parser, value), fuzziness: fuzz}}}, []}}

  defp field_type(_parser, [{LiteralParser, _field_name}, range: :eq, wildcard: "*"]),
    do: {:ok, {%{match_all: %{}}, []}}

  defp field_type(parser, [{LiteralParser, field_name}, range: :eq, wildcard: value]),
    do: {:ok, {%{wildcard: %{field(parser, field_name) => normalize_value(parser, value)}}, []}}


  defp field_type(parser, [{NgramParser, field_name}, range: :eq, literal: value]),
    do: {:ok, {%{match_phrase: %{field(parser, field_name) => normalize_value(parser, value)}}, []}}

  defp field_type(parser, [{NgramParser, field_name}, range: :eq, literal: value, fuzz: _fuzz]),
    do: {:ok, {%{match_phrase: %{field(parser, field_name) => normalize_value(parser, value)}}, []}}

  defp field_type(_parser, [{NgramParser, _field_name}, range: :eq, wildcard: "*"]),
    do: {:ok, {%{match_all: %{}}, []}}

  defp field_type(parser, [{NgramParser, field_name}, range: :eq, wildcard: value]),
    do: {:ok, {%{wildcard: %{field(parser, field_name) => normalize_value(parser, value)}}, []}}


  defp field_type(parser, [{BoolParser, field_name}, range: :eq, bool: value]),
    do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}}

  defp field_type(parser, [{IpParser, field_name}, range: :eq, ip: value]),
    do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}}


  # Types which do support ranges

  defp field_type(parser, [{IntParser, field_name}, range: :eq, int: value]),
    do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}}

  defp field_type(parser, [{IntParser, field_name}, range: :eq, int_range: [lower, upper]]),
    do: {:ok, {%{range: %{field(parser, field_name) => %{gte: lower, lte: upper}}}, []}}

  defp field_type(parser, [{IntParser, field_name}, range: range, int: value]),
    do: {:ok, {%{range: %{field(parser, field_name) => %{range => value}}}, []}}

  defp field_type(_parser, [{IntParser, field_name}, range: _range, int_range: _value]),
    do: {:error, "multiple ranges specified for " <> field_name}


  defp field_type(parser, [{FloatParser, field_name}, range: :eq, float: value]),
    do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}}

  defp field_type(parser, [{FloatParser, field_name}, range: :eq, float_range: [lower, upper]]),
    do: {:ok, {%{range: %{field(parser, field_name) => %{gte: lower, lte: upper}}}, []}}

  defp field_type(parser, [{FloatParser, field_name}, range: range, float: value]),
    do: {:ok, {%{range: %{field(parser, field_name) => %{range => value}}}, []}}

  defp field_type(_parser, [{FloatParser, field_name}, range: _range, float_range: _value]),
    do: {:error, "multiple ranges specified for " <> field_name}


  defp field_type(parser, [{DateParser, field_name}, range: :eq, date: [lower, upper]]),
    do: {:ok, {%{range: %{field(parser, field_name) => %{gte: lower, lt: upper}}}, []}}

  defp field_type(parser, [{DateParser, field_name}, range: r, date: [_lower, upper]]) when r in [:lte, :gt],
    do: {:ok, {%{range: %{field(parser, field_name) => %{r => upper}}}, []}}

  defp field_type(parser, [{DateParser, field_name}, range: r, date: [lower, _upper]]) when r in [:gte, :lt],
    do: {:ok, {%{range: %{field(parser, field_name) => %{r => lower}}}, []}}


  defp field(parser, field_name) do
    parser.aliases[field_name] || field_name
  end

  defp normalize_value(_parser, value) do
    value
    |> String.trim()
    |> String.downcase()
  end

  # Flattens the child of a disjunction or conjunction to improve performance.
  defp flatten_disjunction_child(this_child, %{bool: %{should: next_child}}),
    do: %{bool: %{should: [this_child | next_child]}}
  defp flatten_disjunction_child(this_child, next_child),
    do: %{bool: %{should: [this_child, next_child]}}

  defp flatten_conjunction_child(this_child, %{bool: %{must: next_child}}),
    do: %{bool: %{must: [this_child | next_child]}}
  defp flatten_conjunction_child(this_child, next_child),
    do: %{bool: %{must: [this_child, next_child]}}

  # Flattens the child of a negation to eliminate double negation.
  defp flatten_negation_child(%{bool: %{must_not: next_child}}),
    do: next_child
  defp flatten_negation_child(next_child),
    do: %{bool: %{must_not: next_child}}
end