Merge pull request #299 from philomena-dev/elastix-remove

Remove Elastix and hackney

config/config.exs

@@ -16,9 +16,6 @@ config :logger,
 config :philomena,
   ecto_repos: [Philomena.Repo]
 
-config :elastix,
-  json_codec: Jason
-
 config :exq,
   max_retries: 5,
   scheduler_enable: true,
@@ -37,6 +34,9 @@ config :philomena, PhilomenaWeb.Endpoint,
   render_errors: [view: PhilomenaWeb.ErrorView, accepts: ~w(html json)],
   pubsub_server: Philomena.PubSub
 
+# Configure only SMTP for mailing, not HTTP
+config :swoosh, :api_client, false
+
 # Markdown
 config :philomena, Philomena.Native,
   crate: "philomena",

config/runtime.exs

@@ -88,10 +88,6 @@ config :philomena, :s3_secondary_options,
 
 config :philomena, :s3_secondary_bucket, System.get_env("ALT_S3_BUCKET")
 
-# Don't bail on OpenSearch's self-signed certificate
-config :elastix,
-  httpoison_options: [ssl: [verify: :verify_none]]
-
 config :ex_aws, http_client: PhilomenaMedia.Req
 
 config :ex_aws, :retries,

lib/philomena/comments/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Comments.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/filters/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Filters.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/galleries/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Galleries.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/images/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Images.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/posts/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Posts.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/reports/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Reports.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena/tags/search_index.ex

@@ -1,5 +1,5 @@
 defmodule Philomena.Tags.SearchIndex do
-  @behaviour PhilomenaQuery.SearchIndex
+  @behaviour PhilomenaQuery.Search.Index
 
   @impl true
   def index_name do

lib/philomena_query/search.ex

@@ -10,10 +10,10 @@ defmodule PhilomenaQuery.Search do
   """
 
   alias PhilomenaQuery.Batch
+  alias PhilomenaQuery.Search.Api
   alias Philomena.Repo
   require Logger
   import Ecto.Query
-  import Elastix.HTTP
 
   # todo: fetch through compile_env?
   @policy Philomena.SearchPolicy
@@ -85,11 +85,7 @@ defmodule PhilomenaQuery.Search do
   def create_index!(module) do
     index = @policy.index_for(module)
 
-    Elastix.Index.create(
+    Api.create_index(@policy.opensearch_url(), index.index_name(), index.mapping())
-      @policy.opensearch_url(),
-      index.index_name(),
-      index.mapping()
-    )
   end
 
   @doc ~S"""
@@ -109,7 +105,7 @@ defmodule PhilomenaQuery.Search do
   def delete_index!(module) do
     index = @policy.index_for(module)
 
-    Elastix.Index.delete(@policy.opensearch_url(), index.index_name())
+    Api.delete_index(@policy.opensearch_url(), index.index_name())
   end
 
   @doc ~S"""
@@ -132,9 +128,7 @@ defmodule PhilomenaQuery.Search do
     index_name = index.index_name()
     mapping = index.mapping().mappings.properties
 
-    Elastix.Mapping.put(@policy.opensearch_url(), index_name, "_doc", %{properties: mapping},
+    Api.update_index_mapping(@policy.opensearch_url(), index_name, %{properties: mapping})
-      include_type_name: true
-    )
   end
 
   @doc ~S"""
@@ -157,13 +151,7 @@ defmodule PhilomenaQuery.Search do
     index = @policy.index_for(module)
     data = index.as_json(doc)
 
-    Elastix.Document.index(
+    Api.index_document(@policy.opensearch_url(), index.index_name(), data, data.id)
-      @policy.opensearch_url(),
-      index.index_name(),
-      "_doc",
-      data.id,
-      data
-    )
   end
 
   @doc ~S"""
@@ -186,12 +174,7 @@ defmodule PhilomenaQuery.Search do
   def delete_document(id, module) do
     index = @policy.index_for(module)
 
-    Elastix.Document.delete(
+    Api.delete_document(@policy.opensearch_url(), index.index_name(), id)
-      @policy.opensearch_url(),
-      index.index_name(),
-      "_doc",
-      id
-    )
   end
 
   @doc """
@@ -231,12 +214,7 @@ defmodule PhilomenaQuery.Search do
           ]
         end)
 
-      Elastix.Bulk.post(
+      Api.bulk(@policy.opensearch_url(), lines)
-        @policy.opensearch_url(),
-        lines,
-        index: index.index_name(),
-        httpoison_options: [timeout: 30_000]
-      )
     end)
   end
 
@@ -272,11 +250,6 @@ defmodule PhilomenaQuery.Search do
   def update_by_query(module, query_body, set_replacements, replacements) do
     index = @policy.index_for(module)
 
-    url =
-      @policy.opensearch_url()
-      |> prepare_url([index.index_name(), "_update_by_query"])
-      |> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"})
-
     # "Painless" scripting language
     script = """
       // Replace values in "sets" (arrays in the source document)
@@ -320,7 +293,7 @@ defmodule PhilomenaQuery.Search do
     """
 
     body =
-      Jason.encode!(%{
+      %{
         script: %{
           source: script,
           params: %{
@@ -329,9 +302,9 @@ defmodule PhilomenaQuery.Search do
           }
         },
         query: query_body
-      })
+      }
 
-    {:ok, %{status_code: 200}} = Elastix.HTTP.post(url, body)
+    Api.update_by_query(@policy.opensearch_url(), index.index_name(), body)
   end
 
   @doc ~S"""
@@ -360,13 +333,8 @@ defmodule PhilomenaQuery.Search do
   def search(module, query_body) do
     index = @policy.index_for(module)
 
-    {:ok, %{body: results, status_code: 200}} =
+    {:ok, %{body: results, status: 200}} =
-      Elastix.Search.search(
+      Api.search(@policy.opensearch_url(), index.index_name(), query_body)
-        @policy.opensearch_url(),
-        index.index_name(),
-        [],
-        query_body
-      )
 
     results
   end
@@ -401,13 +369,8 @@ defmodule PhilomenaQuery.Search do
         ]
       end)
 
-    {:ok, %{body: results, status_code: 200}} =
+    {:ok, %{body: results, status: 200}} =
-      Elastix.Search.search(
+      Api.msearch(@policy.opensearch_url(), msearch_body)
-        @policy.opensearch_url(),
-        "_all",
-        [],
-        msearch_body
-      )
 
     results["responses"]
   end

lib/philomena_query/search/api.ex

@@ -0,0 +1,141 @@
+defmodule PhilomenaQuery.Search.Api do
+  @moduledoc """
+  Interaction with OpenSearch API by endpoint name.
+
+  See https://opensearch.org/docs/latest/api-reference for a complete reference.
+  """
+
+  alias PhilomenaQuery.Search.Client
+
+  @type server_url :: String.t()
+  @type index_name :: String.t()
+
+  @type properties :: map()
+  @type mapping :: map()
+  @type document :: map()
+  @type document_id :: integer()
+
+  @doc """
+  Create the index named `name` with the given `mapping`.
+
+  https://opensearch.org/docs/latest/api-reference/index-apis/create-index/
+  """
+  @spec create_index(server_url(), index_name(), mapping()) :: Client.result()
+  def create_index(url, name, mapping) do
+    url
+    |> prepare_url([name])
+    |> Client.put(mapping)
+  end
+
+  @doc """
+  Delete the index named `name`.
+
+  https://opensearch.org/docs/latest/api-reference/index-apis/delete-index/
+  """
+  @spec delete_index(server_url(), index_name()) :: Client.result()
+  def delete_index(url, name) do
+    url
+    |> prepare_url([name])
+    |> Client.delete()
+  end
+
+  @doc """
+  Update the index named `name` with the given `properties`.
+
+  https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/
+  """
+  @spec update_index_mapping(server_url(), index_name(), properties()) :: Client.result()
+  def update_index_mapping(url, name, properties) do
+    url
+    |> prepare_url([name, "_mapping"])
+    |> Client.put(properties)
+  end
+
+  @doc """
+  Index `document` in the index named `name` with integer id `id`.
+
+  https://opensearch.org/docs/latest/api-reference/document-apis/index-document/
+  """
+  @spec index_document(server_url(), index_name(), document(), document_id()) :: Client.result()
+  def index_document(url, name, document, id) do
+    url
+    |> prepare_url([name, "_doc", Integer.to_string(id)])
+    |> Client.put(document)
+  end
+
+  @doc """
+  Remove document in the index named `name` with integer id `id`.
+
+  https://opensearch.org/docs/latest/api-reference/document-apis/delete-document/
+  """
+  @spec delete_document(server_url(), index_name(), document_id()) :: Client.result()
+  def delete_document(url, name, id) do
+    url
+    |> prepare_url([name, "_doc", Integer.to_string(id)])
+    |> Client.delete()
+  end
+
+  @doc """
+  Bulk operation.
+
+  https://opensearch.org/docs/latest/api-reference/document-apis/bulk/
+  """
+  @spec bulk(server_url(), list()) :: Client.result()
+  def bulk(url, lines) do
+    url
+    |> prepare_url(["_bulk"])
+    |> Client.post(lines)
+  end
+
+  @doc """
+  Asynchronous scripted updates.
+
+  Sets `conflicts` to `proceed` and `wait_for_completion` to `false`.
+
+  https://opensearch.org/docs/latest/api-reference/document-apis/update-by-query/
+  """
+  @spec update_by_query(server_url(), index_name(), map()) :: Client.result()
+  def update_by_query(url, name, body) do
+    url
+    |> prepare_url([name, "_update_by_query"])
+    |> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"})
+    |> Client.post(body)
+  end
+
+  @doc """
+  Search for documents in index named `name` with `query`.
+
+  https://opensearch.org/docs/latest/api-reference/search/
+  """
+  @spec search(server_url(), index_name(), map()) :: Client.result()
+  def search(url, name, body) do
+    url
+    |> prepare_url([name, "_search"])
+    |> Client.get(body)
+  end
+
+  @doc """
+  Search for documents in all indices with specified `lines`.
+
+  https://opensearch.org/docs/latest/api-reference/multi-search/
+  """
+  @spec msearch(server_url(), list()) :: Client.result()
+  def msearch(url, lines) do
+    url
+    |> prepare_url(["_msearch"])
+    |> Client.get(lines)
+  end
+
+  @spec prepare_url(String.t(), [String.t()]) :: String.t()
+  defp prepare_url(url, parts) when is_list(parts) do
+    # Combine path generated by the parts with the main URL
+    url
+    |> URI.merge(Path.join(parts))
+    |> to_string()
+  end
+
+  @spec append_query_string(String.t(), map()) :: String.t()
+  defp append_query_string(url, params) do
+    url <> "?" <> URI.encode_query(params)
+  end
+end

lib/philomena_query/search/client.ex

@@ -0,0 +1,62 @@
+defmodule PhilomenaQuery.Search.Client do
+  @moduledoc """
+  HTTP-level interaction with OpenSearch JSON API.
+
+  Allows two styles of parameters for bodies:
+  - map: the map is directly encoded as a JSON object
+  - list: each element of the list is encoded as a JSON object and interspersed with newlines.
+    This is used by bulk APIs.
+  """
+
+  @receive_timeout 30_000
+
+  @type list_or_map :: list() | map()
+  @type result :: {:ok, Req.Response.t()} | {:error, Exception.t()}
+
+  @doc """
+  HTTP GET
+  """
+  @spec get(String.t(), list_or_map()) :: result()
+  def get(url, body) do
+    Req.get(url, encode_options(body))
+  end
+
+  @doc """
+  HTTP POST
+  """
+  @spec post(String.t(), list_or_map()) :: result()
+  def post(url, body) do
+    Req.post(url, encode_options(body))
+  end
+
+  @doc """
+  HTTP PUT
+  """
+  @spec put(String.t(), list_or_map()) :: result()
+  def put(url, body) do
+    Req.put(url, encode_options(body))
+  end
+
+  @doc """
+  HTTP DELETE
+  """
+  @spec delete(String.t()) :: result()
+  def delete(url) do
+    Req.delete(url, encode_options())
+  end
+
+  defp encode_body(body) when is_map(body),
+    do: Jason.encode!(body)
+
+  defp encode_body(body) when is_list(body),
+    do: [Enum.map_intersperse(body, "\n", &Jason.encode!(&1)), "\n"]
+
+  defp encode_options,
+    do: [headers: request_headers(), receive_timeout: @receive_timeout]
+
+  defp encode_options(body),
+    do: Keyword.merge(encode_options(), body: encode_body(body))
+
+  defp request_headers,
+    do: [content_type: "application/json"]
+end

lib/philomena_query/search/index.ex

@@ -1,4 +1,4 @@
-defmodule PhilomenaQuery.SearchIndex do
+defmodule PhilomenaQuery.Search.Index do
   @moduledoc """
   Behaviour module for schemas with search indexing.
   """

mix.exs

@@ -54,7 +54,6 @@ defmodule Philomena.MixProject do
       {:bcrypt_elixir, "~> 3.0"},
       {:pot, "~> 1.0"},
       {:secure_compare, "~> 0.1"},
-      {:elastix, "~> 0.10"},
       {:nimble_parsec, "~> 1.2"},
       {:scrivener_ecto, "~> 2.7"},
       {:pbkdf2, ">= 0.0.0",

mix.lock

@@ -19,7 +19,6 @@
   "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},
   "ecto_network": {:hex, :ecto_network, "1.5.0", "a930c910975e7a91237b858ebf0f4ad7b2aae32fa846275aa203cb858459ec73", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "4d614434ae3e6d373a2f693d56aafaa3f3349714668ffd6d24e760caf578aa2f"},
   "ecto_sql": {:hex, :ecto_sql, "3.11.2", "c7cc7f812af571e50b80294dc2e535821b3b795ce8008d07aa5f336591a185a8", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73c07f995ac17dbf89d3cfaaf688fcefabcd18b7b004ac63b0dc4ef39499ed6b"},
-  "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"},
   "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"},
   "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
@@ -31,9 +30,7 @@
   "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
   "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
   "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"},
-  "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"},
-  "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},