diff --git a/config/config.exs b/config/config.exs index c44f3bf0..12cc771b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -16,9 +16,6 @@ config :logger, config :philomena, ecto_repos: [Philomena.Repo] -config :elastix, - json_codec: Jason - config :exq, max_retries: 5, scheduler_enable: true, @@ -37,6 +34,9 @@ config :philomena, PhilomenaWeb.Endpoint, render_errors: [view: PhilomenaWeb.ErrorView, accepts: ~w(html json)], pubsub_server: Philomena.PubSub +# Configure only SMTP for mailing, not HTTP +config :swoosh, :api_client, false + # Markdown config :philomena, Philomena.Native, crate: "philomena", diff --git a/config/runtime.exs b/config/runtime.exs index 190a1da7..5ebb59e9 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -88,10 +88,6 @@ config :philomena, :s3_secondary_options, config :philomena, :s3_secondary_bucket, System.get_env("ALT_S3_BUCKET") -# Don't bail on OpenSearch's self-signed certificate -config :elastix, - httpoison_options: [ssl: [verify: :verify_none]] - config :ex_aws, http_client: PhilomenaMedia.Req config :ex_aws, :retries, diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index b08b1594..b2358923 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Comments.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/filters/search_index.ex b/lib/philomena/filters/search_index.ex index 12a59385..4d8256a0 100644 --- a/lib/philomena/filters/search_index.ex +++ b/lib/philomena/filters/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Filters.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/galleries/search_index.ex b/lib/philomena/galleries/search_index.ex index 37485b20..b3712cf1 100644 --- a/lib/philomena/galleries/search_index.ex +++ b/lib/philomena/galleries/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Galleries.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/images/search_index.ex b/lib/philomena/images/search_index.ex index 9fb29dc2..2d9265b5 100644 --- a/lib/philomena/images/search_index.ex +++ b/lib/philomena/images/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Images.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/posts/search_index.ex b/lib/philomena/posts/search_index.ex index b0fdb94c..9c8c2780 100644 --- a/lib/philomena/posts/search_index.ex +++ b/lib/philomena/posts/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Posts.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/reports/search_index.ex b/lib/philomena/reports/search_index.ex index 15a08708..61b988de 100644 --- a/lib/philomena/reports/search_index.ex +++ b/lib/philomena/reports/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Reports.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena/tags/search_index.ex b/lib/philomena/tags/search_index.ex index be592d36..ec681a3f 100644 --- a/lib/philomena/tags/search_index.ex +++ b/lib/philomena/tags/search_index.ex @@ -1,5 +1,5 @@ defmodule Philomena.Tags.SearchIndex do - @behaviour PhilomenaQuery.SearchIndex + @behaviour PhilomenaQuery.Search.Index @impl true def index_name do diff --git a/lib/philomena_query/search.ex b/lib/philomena_query/search.ex index 140adf67..2519e580 100644 --- a/lib/philomena_query/search.ex +++ b/lib/philomena_query/search.ex @@ -10,10 +10,10 @@ defmodule PhilomenaQuery.Search do """ alias PhilomenaQuery.Batch + alias PhilomenaQuery.Search.Api alias Philomena.Repo require Logger import Ecto.Query - import Elastix.HTTP # todo: fetch through compile_env? @policy Philomena.SearchPolicy @@ -85,11 +85,7 @@ defmodule PhilomenaQuery.Search do def create_index!(module) do index = @policy.index_for(module) - Elastix.Index.create( - @policy.opensearch_url(), - index.index_name(), - index.mapping() - ) + Api.create_index(@policy.opensearch_url(), index.index_name(), index.mapping()) end @doc ~S""" @@ -109,7 +105,7 @@ defmodule PhilomenaQuery.Search do def delete_index!(module) do index = @policy.index_for(module) - Elastix.Index.delete(@policy.opensearch_url(), index.index_name()) + Api.delete_index(@policy.opensearch_url(), index.index_name()) end @doc ~S""" @@ -132,9 +128,7 @@ defmodule PhilomenaQuery.Search do index_name = index.index_name() mapping = index.mapping().mappings.properties - Elastix.Mapping.put(@policy.opensearch_url(), index_name, "_doc", %{properties: mapping}, - include_type_name: true - ) + Api.update_index_mapping(@policy.opensearch_url(), index_name, %{properties: mapping}) end @doc ~S""" @@ -157,13 +151,7 @@ defmodule PhilomenaQuery.Search do index = @policy.index_for(module) data = index.as_json(doc) - Elastix.Document.index( - @policy.opensearch_url(), - index.index_name(), - "_doc", - data.id, - data - ) + Api.index_document(@policy.opensearch_url(), index.index_name(), data, data.id) end @doc ~S""" @@ -186,12 +174,7 @@ defmodule PhilomenaQuery.Search do def delete_document(id, module) do index = @policy.index_for(module) - Elastix.Document.delete( - @policy.opensearch_url(), - index.index_name(), - "_doc", - id - ) + Api.delete_document(@policy.opensearch_url(), index.index_name(), id) end @doc """ @@ -231,12 +214,7 @@ defmodule PhilomenaQuery.Search do ] end) - Elastix.Bulk.post( - @policy.opensearch_url(), - lines, - index: index.index_name(), - httpoison_options: [timeout: 30_000] - ) + Api.bulk(@policy.opensearch_url(), lines) end) end @@ -272,11 +250,6 @@ defmodule PhilomenaQuery.Search do def update_by_query(module, query_body, set_replacements, replacements) do index = @policy.index_for(module) - url = - @policy.opensearch_url() - |> prepare_url([index.index_name(), "_update_by_query"]) - |> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"}) - # "Painless" scripting language script = """ // Replace values in "sets" (arrays in the source document) @@ -320,7 +293,7 @@ defmodule PhilomenaQuery.Search do """ body = - Jason.encode!(%{ + %{ script: %{ source: script, params: %{ @@ -329,9 +302,9 @@ defmodule PhilomenaQuery.Search do } }, query: query_body - }) + } - {:ok, %{status_code: 200}} = Elastix.HTTP.post(url, body) + Api.update_by_query(@policy.opensearch_url(), index.index_name(), body) end @doc ~S""" @@ -360,13 +333,8 @@ defmodule PhilomenaQuery.Search do def search(module, query_body) do index = @policy.index_for(module) - {:ok, %{body: results, status_code: 200}} = - Elastix.Search.search( - @policy.opensearch_url(), - index.index_name(), - [], - query_body - ) + {:ok, %{body: results, status: 200}} = + Api.search(@policy.opensearch_url(), index.index_name(), query_body) results end @@ -401,13 +369,8 @@ defmodule PhilomenaQuery.Search do ] end) - {:ok, %{body: results, status_code: 200}} = - Elastix.Search.search( - @policy.opensearch_url(), - "_all", - [], - msearch_body - ) + {:ok, %{body: results, status: 200}} = + Api.msearch(@policy.opensearch_url(), msearch_body) results["responses"] end diff --git a/lib/philomena_query/search/api.ex b/lib/philomena_query/search/api.ex new file mode 100644 index 00000000..01850a82 --- /dev/null +++ b/lib/philomena_query/search/api.ex @@ -0,0 +1,141 @@ +defmodule PhilomenaQuery.Search.Api do + @moduledoc """ + Interaction with OpenSearch API by endpoint name. + + See https://opensearch.org/docs/latest/api-reference for a complete reference. + """ + + alias PhilomenaQuery.Search.Client + + @type server_url :: String.t() + @type index_name :: String.t() + + @type properties :: map() + @type mapping :: map() + @type document :: map() + @type document_id :: integer() + + @doc """ + Create the index named `name` with the given `mapping`. + + https://opensearch.org/docs/latest/api-reference/index-apis/create-index/ + """ + @spec create_index(server_url(), index_name(), mapping()) :: Client.result() + def create_index(url, name, mapping) do + url + |> prepare_url([name]) + |> Client.put(mapping) + end + + @doc """ + Delete the index named `name`. + + https://opensearch.org/docs/latest/api-reference/index-apis/delete-index/ + """ + @spec delete_index(server_url(), index_name()) :: Client.result() + def delete_index(url, name) do + url + |> prepare_url([name]) + |> Client.delete() + end + + @doc """ + Update the index named `name` with the given `properties`. + + https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/ + """ + @spec update_index_mapping(server_url(), index_name(), properties()) :: Client.result() + def update_index_mapping(url, name, properties) do + url + |> prepare_url([name, "_mapping"]) + |> Client.put(properties) + end + + @doc """ + Index `document` in the index named `name` with integer id `id`. + + https://opensearch.org/docs/latest/api-reference/document-apis/index-document/ + """ + @spec index_document(server_url(), index_name(), document(), document_id()) :: Client.result() + def index_document(url, name, document, id) do + url + |> prepare_url([name, "_doc", Integer.to_string(id)]) + |> Client.put(document) + end + + @doc """ + Remove document in the index named `name` with integer id `id`. + + https://opensearch.org/docs/latest/api-reference/document-apis/delete-document/ + """ + @spec delete_document(server_url(), index_name(), document_id()) :: Client.result() + def delete_document(url, name, id) do + url + |> prepare_url([name, "_doc", Integer.to_string(id)]) + |> Client.delete() + end + + @doc """ + Bulk operation. + + https://opensearch.org/docs/latest/api-reference/document-apis/bulk/ + """ + @spec bulk(server_url(), list()) :: Client.result() + def bulk(url, lines) do + url + |> prepare_url(["_bulk"]) + |> Client.post(lines) + end + + @doc """ + Asynchronous scripted updates. + + Sets `conflicts` to `proceed` and `wait_for_completion` to `false`. + + https://opensearch.org/docs/latest/api-reference/document-apis/update-by-query/ + """ + @spec update_by_query(server_url(), index_name(), map()) :: Client.result() + def update_by_query(url, name, body) do + url + |> prepare_url([name, "_update_by_query"]) + |> append_query_string(%{conflicts: "proceed", wait_for_completion: "false"}) + |> Client.post(body) + end + + @doc """ + Search for documents in index named `name` with `query`. + + https://opensearch.org/docs/latest/api-reference/search/ + """ + @spec search(server_url(), index_name(), map()) :: Client.result() + def search(url, name, body) do + url + |> prepare_url([name, "_search"]) + |> Client.get(body) + end + + @doc """ + Search for documents in all indices with specified `lines`. + + https://opensearch.org/docs/latest/api-reference/multi-search/ + """ + @spec msearch(server_url(), list()) :: Client.result() + def msearch(url, lines) do + url + |> prepare_url(["_msearch"]) + |> Client.get(lines) + end + + @spec prepare_url(String.t(), [String.t()]) :: String.t() + defp prepare_url(url, parts) when is_list(parts) do + # Combine path generated by the parts with the main URL + url + |> URI.merge(Path.join(parts)) + |> to_string() + end + + @spec append_query_string(String.t(), map()) :: String.t() + defp append_query_string(url, params) do + url <> "?" <> URI.encode_query(params) + end +end diff --git a/lib/philomena_query/search/client.ex b/lib/philomena_query/search/client.ex new file mode 100644 index 00000000..98a81820 --- /dev/null +++ b/lib/philomena_query/search/client.ex @@ -0,0 +1,62 @@ +defmodule PhilomenaQuery.Search.Client do + @moduledoc """ + HTTP-level interaction with OpenSearch JSON API. + + Allows two styles of parameters for bodies: + - map: the map is directly encoded as a JSON object + - list: each element of the list is encoded as a JSON object and interspersed with newlines. + This is used by bulk APIs. + """ + + @receive_timeout 30_000 + + @type list_or_map :: list() | map() + @type result :: {:ok, Req.Response.t()} | {:error, Exception.t()} + + @doc """ + HTTP GET + """ + @spec get(String.t(), list_or_map()) :: result() + def get(url, body) do + Req.get(url, encode_options(body)) + end + + @doc """ + HTTP POST + """ + @spec post(String.t(), list_or_map()) :: result() + def post(url, body) do + Req.post(url, encode_options(body)) + end + + @doc """ + HTTP PUT + """ + @spec put(String.t(), list_or_map()) :: result() + def put(url, body) do + Req.put(url, encode_options(body)) + end + + @doc """ + HTTP DELETE + """ + @spec delete(String.t()) :: result() + def delete(url) do + Req.delete(url, encode_options()) + end + + defp encode_body(body) when is_map(body), + do: Jason.encode!(body) + + defp encode_body(body) when is_list(body), + do: [Enum.map_intersperse(body, "\n", &Jason.encode!(&1)), "\n"] + + defp encode_options, + do: [headers: request_headers(), receive_timeout: @receive_timeout] + + defp encode_options(body), + do: Keyword.merge(encode_options(), body: encode_body(body)) + + defp request_headers, + do: [content_type: "application/json"] +end diff --git a/lib/philomena_query/search_index.ex b/lib/philomena_query/search/index.ex similarity index 96% rename from lib/philomena_query/search_index.ex rename to lib/philomena_query/search/index.ex index 119d2613..90f88741 100644 --- a/lib/philomena_query/search_index.ex +++ b/lib/philomena_query/search/index.ex @@ -1,4 +1,4 @@ -defmodule PhilomenaQuery.SearchIndex do +defmodule PhilomenaQuery.Search.Index do @moduledoc """ Behaviour module for schemas with search indexing. """ diff --git a/mix.exs b/mix.exs index 6286042d..05f26cc9 100644 --- a/mix.exs +++ b/mix.exs @@ -54,7 +54,6 @@ defmodule Philomena.MixProject do {:bcrypt_elixir, "~> 3.0"}, {:pot, "~> 1.0"}, {:secure_compare, "~> 0.1"}, - {:elastix, "~> 0.10"}, {:nimble_parsec, "~> 1.2"}, {:scrivener_ecto, "~> 2.7"}, {:pbkdf2, ">= 0.0.0", diff --git a/mix.lock b/mix.lock index 389ea193..4d006ee9 100644 --- a/mix.lock +++ b/mix.lock @@ -19,7 +19,6 @@ "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "ecto_network": {:hex, :ecto_network, "1.5.0", "a930c910975e7a91237b858ebf0f4ad7b2aae32fa846275aa203cb858459ec73", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "4d614434ae3e6d373a2f693d56aafaa3f3349714668ffd6d24e760caf578aa2f"}, "ecto_sql": {:hex, :ecto_sql, "3.11.2", "c7cc7f812af571e50b80294dc2e535821b3b795ce8008d07aa5f336591a185a8", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73c07f995ac17dbf89d3cfaaf688fcefabcd18b7b004ac63b0dc4ef39499ed6b"}, - "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"}, "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, @@ -31,9 +30,7 @@ "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "hpax": {:hex, :hpax, "0.2.0", "5a58219adcb75977b2edce5eb22051de9362f08236220c9e859a47111c194ff5", [:mix], [], "hexpm", "bea06558cdae85bed075e6c036993d43cd54d447f76d8190a8db0dc5893fa2f1"}, - "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},