From 620f316db87518edd9c39ec670694bc285fb2524 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Tue, 24 Dec 2019 16:14:42 -0500 Subject: [PATCH] break compile time dependencies for elasticsearch index usage --- lib/mix/tasks/reindex_all.ex | 15 +- lib/philomena/batch.ex | 55 +++++ lib/philomena/comments.ex | 5 +- lib/philomena/comments/comment.ex | 5 - ...lasticsearch.ex => elasticsearch_index.ex} | 16 +- lib/philomena/elasticsearch.ex | 229 +++++++++--------- lib/philomena/elasticsearch_index.ex | 18 ++ lib/philomena/galleries.ex | 5 +- ...lasticsearch.ex => elasticsearch_index.ex} | 17 +- lib/philomena/galleries/gallery.ex | 5 - lib/philomena/image_navigator.ex | 10 +- lib/philomena/images.ex | 3 +- ...lasticsearch.ex => elasticsearch_index.ex} | 20 +- lib/philomena/images/image.ex | 5 - lib/philomena/posts.ex | 3 +- ...lasticsearch.ex => elasticsearch_index.ex} | 19 +- lib/philomena/posts/post.ex | 5 - lib/philomena/reports.ex | 3 +- ...lasticsearch.ex => elasticsearch_index.ex} | 16 +- lib/philomena/reports/report.ex | 5 - lib/philomena/tags.ex | 9 +- ...lasticsearch.ex => elasticsearch_index.ex} | 19 +- lib/philomena/tags/tag.ex | 5 - .../controllers/activity_controller.ex | 4 +- .../controllers/admin/report_controller.ex | 4 +- .../api/json/search/comment_controller.ex | 6 +- .../api/json/search/gallery_controller.ex | 6 +- .../api/json/search/post_controller.ex | 6 +- .../api/json/search/tag_controller.ex | 6 +- .../controllers/api/rss/watched_controller.ex | 31 +-- .../controllers/comment_controller.ex | 4 +- .../controllers/gallery_controller.ex | 4 +- .../controllers/image/random_controller.ex | 6 +- .../controllers/post_controller.ex | 4 +- .../controllers/profile_controller.ex | 7 +- .../controllers/stat_controller.ex | 6 +- .../tag/autocomplete_controller.ex | 6 +- .../controllers/tag_controller.ex | 4 +- lib/philomena_web/image_json.ex | 4 +- lib/philomena_web/image_loader.ex | 4 +- lib/philomena_web/views/tag_view.ex | 4 +- 41 files changed, 374 insertions(+), 234 deletions(-) create mode 100644 lib/philomena/batch.ex rename lib/philomena/comments/{elasticsearch.ex => elasticsearch_index.ex} (85%) create mode 100644 lib/philomena/elasticsearch_index.ex rename lib/philomena/galleries/{elasticsearch.ex => elasticsearch_index.ex} (84%) rename lib/philomena/images/{elasticsearch.ex => elasticsearch_index.ex} (96%) rename lib/philomena/posts/{elasticsearch.ex => elasticsearch_index.ex} (89%) rename lib/philomena/reports/{elasticsearch.ex => elasticsearch_index.ex} (88%) rename lib/philomena/tags/{elasticsearch.ex => elasticsearch_index.ex} (90%) diff --git a/lib/mix/tasks/reindex_all.ex b/lib/mix/tasks/reindex_all.ex index 9ed4c10e..bdf3b42a 100644 --- a/lib/mix/tasks/reindex_all.ex +++ b/lib/mix/tasks/reindex_all.ex @@ -1,6 +1,7 @@ defmodule Mix.Tasks.ReindexAll do use Mix.Task + alias Philomena.Elasticsearch alias Philomena.{Comments.Comment, Galleries.Gallery, Posts.Post, Images.Image, Reports.Report, Tags.Tag} alias Philomena.{Comments, Galleries, Posts, Images, Tags} alias Philomena.Polymorphic @@ -16,21 +17,21 @@ defmodule Mix.Tasks.ReindexAll do {:ok, _apps} = Application.ensure_all_started(:philomena) for {context, schema} <- [{Images, Image}, {Comments, Comment}, {Galleries, Gallery}, {Tags, Tag}, {Posts, Post}] do - schema.delete_index! - schema.create_index! + Elasticsearch.delete_index!(schema) + Elasticsearch.create_index!(schema) - schema.reindex(schema |> preload(^context.indexing_preloads())) + Elasticsearch.reindex(preload(schema, ^context.indexing_preloads()), schema) end # Reports are a bit special - Report.delete_index! - Report.create_index! + Elasticsearch.delete_index!(Report) + Elasticsearch.create_index!(Report) Report |> preload([:user, :admin]) |> Repo.all() |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type]) - |> Enum.map(&Report.index_document/1) + |> Enum.map(&Elasticsearch.index_document(&1, Report)) end -end \ No newline at end of file +end diff --git a/lib/philomena/batch.ex b/lib/philomena/batch.ex new file mode 100644 index 00000000..72e996db --- /dev/null +++ b/lib/philomena/batch.ex @@ -0,0 +1,55 @@ +defmodule Philomena.Batch do + alias Philomena.Repo + import Ecto.Query + + @doc """ + Load records from the given queryable in batches, to avoid locking. + + Valid options: + * :batch_size + * :id_field + """ + def record_batches(queryable, opts \\ [], callback) do + query_batches(queryable, opts, &callback.(Repo.all(&1))) + end + + @doc """ + Load queries from the given queryable in batches, to avoid locking. + + Valid options: + * :batch_size + * :id_field + """ + def query_batches(queryable, opts \\ [], callback) do + ids = load_ids(queryable, -1, opts) + + query_batches(queryable, opts, callback, ids) + end + + defp query_batches(_queryable, _opts, _callback, []), do: [] + defp query_batches(queryable, opts, callback, ids) do + id_field = Keyword.get(opts, :id_field, :id) + + queryable + |> where([m], field(m, ^id_field) in ^ids) + |> callback.() + + ids = load_ids(queryable, Enum.max(ids), opts) + + query_batches(queryable, opts, callback, ids) + end + + defp load_ids(queryable, max_id, opts) do + id_field = Keyword.get(opts, :id_field, :id) + batch_size = Keyword.get(opts, :batch_size, 1000) + + queryable + |> exclude(:preload) + |> exclude(:order_by) + |> order_by(asc: ^id_field) + |> where([m], field(m, ^id_field) > ^max_id) + |> select([m], field(m, ^id_field)) + |> limit(^batch_size) + |> Repo.all() + end +end diff --git a/lib/philomena/comments.ex b/lib/philomena/comments.ex index 98c9dc8c..bfd02b11 100644 --- a/lib/philomena/comments.ex +++ b/lib/philomena/comments.ex @@ -7,6 +7,7 @@ defmodule Philomena.Comments do alias Ecto.Multi alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Reports.Report alias Philomena.Comments.Comment alias Philomena.Images.Image @@ -194,7 +195,7 @@ defmodule Philomena.Comments do |> preload(^indexing_preloads()) |> where(id: ^comment.id) |> Repo.one() - |> Comment.index_document() + |> Elasticsearch.index_document(Comment) end comment @@ -205,7 +206,7 @@ defmodule Philomena.Comments do Comment |> preload(^indexing_preloads()) |> where(image_id: ^image.id) - |> Comment.reindex() + |> Elasticsearch.reindex(Comment) end image diff --git a/lib/philomena/comments/comment.ex b/lib/philomena/comments/comment.ex index 01cf2e72..7852a87d 100644 --- a/lib/philomena/comments/comment.ex +++ b/lib/philomena/comments/comment.ex @@ -2,11 +2,6 @@ defmodule Philomena.Comments.Comment do use Ecto.Schema import Ecto.Changeset - use Philomena.Elasticsearch, - definition: Philomena.Comments.Elasticsearch, - index_name: "comments", - doc_type: "comment" - alias Philomena.Images.Image alias Philomena.Users.User diff --git a/lib/philomena/comments/elasticsearch.ex b/lib/philomena/comments/elasticsearch_index.ex similarity index 85% rename from lib/philomena/comments/elasticsearch.ex rename to lib/philomena/comments/elasticsearch_index.ex index a65cf899..421def79 100644 --- a/lib/philomena/comments/elasticsearch.ex +++ b/lib/philomena/comments/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Comments.Elasticsearch do +defmodule Philomena.Comments.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "comments" + end + + @impl true + def doc_type do + "comment" + end + + @impl true def mapping do %{ settings: %{ @@ -29,6 +42,7 @@ defmodule Philomena.Comments.Elasticsearch do } end + @impl true def as_json(comment) do %{ id: comment.id, diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex index f55a1dd7..b33d13c7 100644 --- a/lib/philomena/elasticsearch.ex +++ b/lib/philomena/elasticsearch.ex @@ -1,141 +1,142 @@ defmodule Philomena.Elasticsearch do - defmacro __using__(opts) do - definition = Keyword.fetch!(opts, :definition) - index_name = Keyword.fetch!(opts, :index_name) - doc_type = Keyword.fetch!(opts, :doc_type) + alias Philomena.Batch + alias Philomena.Repo + require Logger + import Ecto.Query - elastic_url = Application.get_env(:philomena, :elasticsearch_url) + alias Philomena.Comments.Comment + alias Philomena.Galleries.Gallery + alias Philomena.Images.Image + alias Philomena.Posts.Post + alias Philomena.Reports.Report + alias Philomena.Tags.Tag - quote do - alias Philomena.Repo - import Ecto.Query, warn: false - require Logger + alias Philomena.Comments.ElasticsearchIndex, as: CommentIndex + alias Philomena.Galleries.ElasticsearchIndex, as: GalleryIndex + alias Philomena.Images.ElasticsearchIndex, as: ImageIndex + alias Philomena.Posts.ElasticsearchIndex, as: PostIndex + alias Philomena.Reports.ElasticsearchIndex, as: ReportIndex + alias Philomena.Tags.ElasticsearchIndex, as: TagIndex - def create_index! do - Elastix.Index.create( - unquote(elastic_url), - unquote(index_name), - unquote(definition).mapping() - ) - end + defp index_for(Comment), do: CommentIndex + defp index_for(Gallery), do: GalleryIndex + defp index_for(Image), do: ImageIndex + defp index_for(Post), do: PostIndex + defp index_for(Report), do: ReportIndex + defp index_for(Tag), do: TagIndex - def delete_index! do - Elastix.Index.delete(unquote(elastic_url), unquote(index_name)) - end + defp elastic_url do + Application.get_env(:philomena, :elasticsearch_url) + end - def index_document(doc) do - data = unquote(definition).as_json(doc) + def create_index!(module) do + index = index_for(module) - Elastix.Document.index( - unquote(elastic_url), - unquote(index_name), - [unquote(doc_type)], - data.id, - data - ) - end + Elastix.Index.create( + elastic_url(), + index.index_name(), + index.mapping() + ) + end - def delete_document(id) do - Elastix.Document.delete( - unquote(elastic_url), - unquote(index_name), - unquote(doc_type), - id - ) - end + def delete_index!(module) do + index = index_for(module) - def reindex(ecto_query, batch_size \\ 1000) do - ids = - ecto_query - |> exclude(:preload) - |> exclude(:order_by) - |> order_by(asc: :id) - |> select([m], m.id) - |> limit(^batch_size) - |> Repo.all() + Elastix.Index.delete(elastic_url(), index.index_name()) + end - reindex(ecto_query, batch_size, ids) - end + def index_document(doc, module) do + index = index_for(module) + data = index.as_json(doc) - def reindex(ecto_query, batch_size, []), do: nil + Elastix.Document.index( + elastic_url(), + index.index_name(), + [index.doc_type()], + data.id, + data + ) + end - def reindex(ecto_query, batch_size, ids) do - lines = - ecto_query - |> where([m], m.id in ^ids) - |> Repo.all() - |> Enum.flat_map(fn m -> - doc = unquote(definition).as_json(m) + def delete_document(id, module) do + index = index_for(module) - [ - %{index: %{_index: unquote(index_name), _type: unquote(doc_type), _id: doc.id}}, - doc - ] - end) + Elastix.Document.delete( + elastic_url(), + index.index_name(), + index.doc_type(), + id + ) + end - Elastix.Bulk.post(unquote(elastic_url), lines, - index: unquote(index_name), - httpoison_options: [timeout: 30_000] - ) + def reindex(queryable, module, opts \\ []) do + index = index_for(module) - ids = - ecto_query - |> exclude(:preload) - |> exclude(:order_by) - |> order_by(asc: :id) - |> where([m], m.id > ^Enum.max(ids)) - |> select([m], m.id) - |> limit(^batch_size) - |> Repo.all() + Batch.record_batches(queryable, opts, fn records -> + lines = + Enum.flat_map(records, fn record -> + doc = index.as_json(record) - reindex(ecto_query, batch_size, ids) - end + [ + %{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}}, + doc + ] + end) - def search(query_body) do - {:ok, %{body: results, status_code: 200}} = - Elastix.Search.search( - unquote(elastic_url), - unquote(index_name), - [unquote(doc_type)], - query_body - ) + Elastix.Bulk.post( + elastic_url(), + lines, + index: index.index_name(), + httpoison_options: [timeout: 30_000] + ) + end) + end - results - end + def search(module, query_body) do + index = index_for(module) - def search_results(elastic_query, pagination_params \\ %{}) do - page_number = pagination_params[:page_number] || 1 - page_size = pagination_params[:page_size] || 25 - elastic_query = Map.merge(elastic_query, %{from: (page_number - 1) * page_size, size: page_size, _source: false}) + {:ok, %{body: results, status_code: 200}} = + Elastix.Search.search( + elastic_url(), + index.index_name(), + [index.doc_type()], + query_body + ) - results = search(elastic_query) - time = results["took"] - count = results["hits"]["total"] - entries = results["hits"]["hits"] |> Enum.map(&String.to_integer(&1["_id"])) + results + end - Logger.debug("[Elasticsearch] Query took #{time}ms") + def search_results(module, elastic_query, pagination_params \\ %{}) do + page_number = pagination_params[:page_number] || 1 + page_size = pagination_params[:page_size] || 25 + elastic_query = Map.merge(elastic_query, %{from: (page_number - 1) * page_size, size: page_size, _source: false}) - %Scrivener.Page{ - entries: entries, - page_number: page_number, - page_size: page_size, - total_entries: count, - total_pages: div(count + page_size - 1, page_size) - } - end + results = search(module, elastic_query) + time = results["took"] + count = results["hits"]["total"] + entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"])) - def search_records(elastic_query, pagination_params \\ %{}, ecto_query \\ __MODULE__) do - page = search_results(elastic_query, pagination_params) - ids = page.entries + Logger.debug("[Elasticsearch] Query took #{time}ms") - records = - ecto_query - |> where([m], m.id in ^ids) - |> Repo.all() - |> Enum.sort_by(&Enum.find_index(ids, fn el -> el == &1.id end)) + %Scrivener.Page{ + entries: entries, + page_number: page_number, + page_size: page_size, + total_entries: count, + total_pages: div(count + page_size - 1, page_size) + } + end - %{page | entries: records} - end - end + def search_records(module, elastic_query, pagination_params \\ %{}, ecto_query \\ __MODULE__) do + page = search_results(module, elastic_query, pagination_params) + ids = page.entries + + records = + ecto_query + |> where([m], m.id in ^ids) + |> Repo.all() + |> Enum.sort_by(&Enum.find_index(ids, fn el -> el == &1.id end)) + + %{page | entries: records} end end diff --git a/lib/philomena/elasticsearch_index.ex b/lib/philomena/elasticsearch_index.ex new file mode 100644 index 00000000..e488359d --- /dev/null +++ b/lib/philomena/elasticsearch_index.ex @@ -0,0 +1,18 @@ +defmodule Philomena.ElasticsearchIndex do + # Returns the index name for the index. + # This is usually a collection name like "images". + @callback index_name() :: String.t() + + # Returns the document type for the index. + # This is usually the singular of the index name, like "image". + # + # TODO: Remove for ES 7.0 + @callback doc_type() :: String.t() + + # Returns the mapping and settings for the index. + @callback mapping() :: map() + + # Returns the JSON representation of the given struct + # for indexing in Elasticsearch. + @callback as_json(struct()) :: map() +end diff --git a/lib/philomena/galleries.ex b/lib/philomena/galleries.ex index 22e74fea..7011af4a 100644 --- a/lib/philomena/galleries.ex +++ b/lib/philomena/galleries.ex @@ -7,6 +7,7 @@ defmodule Philomena.Galleries do alias Ecto.Multi alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Galleries.Gallery alias Philomena.Galleries.Interaction alias Philomena.Notifications @@ -127,7 +128,7 @@ defmodule Philomena.Galleries do |> preload(^indexing_preloads()) |> where(id: ^gallery.id) |> Repo.one() - |> Gallery.index_document() + |> Elasticsearch.index_document(Gallery) end gallery @@ -135,7 +136,7 @@ defmodule Philomena.Galleries do def unindex_gallery(%Gallery{} = gallery) do spawn fn -> - Gallery.delete_document(gallery.id) + Elasticsearch.delete_document(gallery.id, Gallery) end gallery diff --git a/lib/philomena/galleries/elasticsearch.ex b/lib/philomena/galleries/elasticsearch_index.ex similarity index 84% rename from lib/philomena/galleries/elasticsearch.ex rename to lib/philomena/galleries/elasticsearch_index.ex index 98f6b18d..9a0b0d79 100644 --- a/lib/philomena/galleries/elasticsearch.ex +++ b/lib/philomena/galleries/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Galleries.Elasticsearch do +defmodule Philomena.Galleries.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "galleries" + end + + @impl true + def doc_type do + "gallery" + end + + @impl true def mapping do %{ settings: %{ @@ -28,7 +41,7 @@ defmodule Philomena.Galleries.Elasticsearch do } end - # [:subscribers, :creator, :interactions] + @impl true def as_json(gallery) do %{ id: gallery.id, diff --git a/lib/philomena/galleries/gallery.ex b/lib/philomena/galleries/gallery.ex index 32a0cc10..41c02609 100644 --- a/lib/philomena/galleries/gallery.ex +++ b/lib/philomena/galleries/gallery.ex @@ -2,11 +2,6 @@ defmodule Philomena.Galleries.Gallery do use Ecto.Schema import Ecto.Changeset - use Philomena.Elasticsearch, - definition: Philomena.Galleries.Elasticsearch, - index_name: "galleries", - doc_type: "gallery" - alias Philomena.Images.Image alias Philomena.Users.User alias Philomena.Galleries.Interaction diff --git a/lib/philomena/image_navigator.ex b/lib/philomena/image_navigator.ex index 68effae9..d5874e8b 100644 --- a/lib/philomena/image_navigator.ex +++ b/lib/philomena/image_navigator.ex @@ -1,6 +1,7 @@ defmodule Philomena.ImageNavigator do alias Philomena.ImageSorter - alias Philomena.Images.{Image, Elasticsearch} + alias Philomena.Images.{Image, ElasticsearchIndex} + alias Philomena.Elasticsearch alias Philomena.Repo import Ecto.Query @@ -30,7 +31,7 @@ defmodule Philomena.ImageNavigator do |> preload(:gallery_interactions) |> Repo.one() |> Map.merge(empty_fields()) - |> Elasticsearch.as_json() + |> ElasticsearchIndex.as_json() sort_data = ImageSorter.parse_sort(params) @@ -42,7 +43,8 @@ defmodule Philomena.ImageNavigator do sorts = sortify(sorts, image_index) filters = filterify(filters, image_index) - Image.search_records( + Elasticsearch.search_records( + Image, %{ query: %{ bool: %{ @@ -168,4 +170,4 @@ defmodule Philomena.ImageNavigator do defp parse_val(%NaiveDateTime{} = value), do: NaiveDateTime.to_iso8601(value) defp parse_val(value), do: value -end \ No newline at end of file +end diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex index eb529172..1d4a0fe5 100644 --- a/lib/philomena/images.ex +++ b/lib/philomena/images.ex @@ -8,6 +8,7 @@ defmodule Philomena.Images do alias Ecto.Multi alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Images.Image alias Philomena.Images.Hider alias Philomena.Images.Uploader @@ -483,7 +484,7 @@ defmodule Philomena.Images do Image |> preload(^indexing_preloads()) |> where([i], i.id in ^image_ids) - |> Image.reindex() + |> Elasticsearch.reindex(Image) end image_ids diff --git a/lib/philomena/images/elasticsearch.ex b/lib/philomena/images/elasticsearch_index.ex similarity index 96% rename from lib/philomena/images/elasticsearch.ex rename to lib/philomena/images/elasticsearch_index.ex index d1b7a482..44be16e8 100644 --- a/lib/philomena/images/elasticsearch.ex +++ b/lib/philomena/images/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Images.Elasticsearch do +defmodule Philomena.Images.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "images" + end + + @impl true + def doc_type do + "image" + end + + @impl true def mapping do %{ settings: %{ @@ -77,10 +90,7 @@ defmodule Philomena.Images.Elasticsearch do } end - # preload([ - # :user, :deleter, :gallery_interactions, :upvoters, :downvoters, :favers, - # :hiders, tags: [:aliases, :aliased_tag] - # ]) + @impl true def as_json(image) do %{ id: image.id, diff --git a/lib/philomena/images/image.ex b/lib/philomena/images/image.ex index 23e91b7e..e903d168 100644 --- a/lib/philomena/images/image.ex +++ b/lib/philomena/images/image.ex @@ -1,11 +1,6 @@ defmodule Philomena.Images.Image do use Ecto.Schema - use Philomena.Elasticsearch, - definition: Philomena.Images.Elasticsearch, - index_name: "images", - doc_type: "image" - import Ecto.Changeset import Ecto.Query diff --git a/lib/philomena/posts.ex b/lib/philomena/posts.ex index 8ae46a20..332c87a1 100644 --- a/lib/philomena/posts.ex +++ b/lib/philomena/posts.ex @@ -7,6 +7,7 @@ defmodule Philomena.Posts do alias Ecto.Multi alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Topics.Topic alias Philomena.Topics alias Philomena.Posts.Post @@ -207,7 +208,7 @@ defmodule Philomena.Posts do |> preload(^indexing_preloads()) |> where(id: ^post.id) |> Repo.one() - |> Post.index_document() + |> Elasticsearch.index_document(Post) end post diff --git a/lib/philomena/posts/elasticsearch.ex b/lib/philomena/posts/elasticsearch_index.ex similarity index 89% rename from lib/philomena/posts/elasticsearch.ex rename to lib/philomena/posts/elasticsearch_index.ex index ebc0778e..4644678d 100644 --- a/lib/philomena/posts/elasticsearch.ex +++ b/lib/philomena/posts/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Posts.Elasticsearch do +defmodule Philomena.Posts.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "posts" + end + + @impl true + def doc_type do + "post" + end + + @impl true def mapping do %{ settings: %{ @@ -36,7 +49,7 @@ defmodule Philomena.Posts.Elasticsearch do } end - # [:user, topic: :forum] + @impl true def as_json(post) do %{ id: post.id, @@ -59,4 +72,4 @@ defmodule Philomena.Posts.Elasticsearch do destroyed_content: post.destroyed_content } end -end \ No newline at end of file +end diff --git a/lib/philomena/posts/post.ex b/lib/philomena/posts/post.ex index f2df618d..77ebbac2 100644 --- a/lib/philomena/posts/post.ex +++ b/lib/philomena/posts/post.ex @@ -2,11 +2,6 @@ defmodule Philomena.Posts.Post do use Ecto.Schema import Ecto.Changeset - use Philomena.Elasticsearch, - definition: Philomena.Posts.Elasticsearch, - index_name: "posts", - doc_type: "post" - alias Philomena.Users.User alias Philomena.Topics.Topic diff --git a/lib/philomena/reports.ex b/lib/philomena/reports.ex index 88fb913a..07e17c72 100644 --- a/lib/philomena/reports.ex +++ b/lib/philomena/reports.ex @@ -6,6 +6,7 @@ defmodule Philomena.Reports do import Ecto.Query, warn: false alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Reports.Report alias Philomena.Polymorphic @@ -128,7 +129,7 @@ defmodule Philomena.Reports do |> preload([:user, :admin]) |> Repo.all() |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type]) - |> Enum.map(&Report.index_document/1) + |> Enum.map(&Elasticsearch.index_document(&1, Report)) end report_ids diff --git a/lib/philomena/reports/elasticsearch.ex b/lib/philomena/reports/elasticsearch_index.ex similarity index 88% rename from lib/philomena/reports/elasticsearch.ex rename to lib/philomena/reports/elasticsearch_index.ex index 4aeec056..6a953fdd 100644 --- a/lib/philomena/reports/elasticsearch.ex +++ b/lib/philomena/reports/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Reports.Elasticsearch do +defmodule Philomena.Reports.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "reports" + end + + @impl true + def doc_type do + "report" + end + + @impl true def mapping do %{ settings: %{ @@ -32,6 +45,7 @@ defmodule Philomena.Reports.Elasticsearch do } end + @impl true def as_json(report) do %{ id: report.id, diff --git a/lib/philomena/reports/report.ex b/lib/philomena/reports/report.ex index 8199f441..f64fd754 100644 --- a/lib/philomena/reports/report.ex +++ b/lib/philomena/reports/report.ex @@ -2,11 +2,6 @@ defmodule Philomena.Reports.Report do use Ecto.Schema import Ecto.Changeset - use Philomena.Elasticsearch, - definition: Philomena.Reports.Elasticsearch, - index_name: "reports", - doc_type: "report" - alias Philomena.Users.User schema "reports" do diff --git a/lib/philomena/tags.ex b/lib/philomena/tags.ex index 8feb17e2..b1ed1a82 100644 --- a/lib/philomena/tags.ex +++ b/lib/philomena/tags.ex @@ -7,6 +7,7 @@ defmodule Philomena.Tags do alias Ecto.Multi alias Philomena.Repo + alias Philomena.Elasticsearch alias Philomena.Tags.Tag alias Philomena.Tags.Uploader alias Philomena.Images @@ -165,12 +166,12 @@ defmodule Philomena.Tags do {:ok, tag} = Repo.delete(tag) - Tag.delete_document(tag.id) + Elasticsearch.delete_document(tag.id, Tag) Image |> where([i], i.id in ^image_ids) |> preload(^Images.indexing_preloads()) - |> Image.reindex() + |> Elasticsearch.reindex(Image) end def alias_tag(%Tag{} = tag, attrs) do @@ -234,7 +235,7 @@ defmodule Philomena.Tags do |> join(:inner, [i], _ in assoc(i, :tags)) |> where([_i, t], t.id == ^tag.id) |> preload(^Images.indexing_preloads()) - |> Image.reindex() + |> Elasticsearch.reindex(Image) end def unalias_tag(%Tag{} = tag) do @@ -316,7 +317,7 @@ defmodule Philomena.Tags do Tag |> preload(^indexing_preloads()) |> where([t], t.id in ^ids) - |> Tag.reindex() + |> Elasticsearch.reindex(Tag) end tags diff --git a/lib/philomena/tags/elasticsearch.ex b/lib/philomena/tags/elasticsearch_index.ex similarity index 90% rename from lib/philomena/tags/elasticsearch.ex rename to lib/philomena/tags/elasticsearch_index.ex index 7cce7184..ccea5b27 100644 --- a/lib/philomena/tags/elasticsearch.ex +++ b/lib/philomena/tags/elasticsearch_index.ex @@ -1,4 +1,17 @@ -defmodule Philomena.Tags.Elasticsearch do +defmodule Philomena.Tags.ElasticsearchIndex do + @behaviour Philomena.ElasticsearchIndex + + @impl true + def index_name do + "tags" + end + + @impl true + def doc_type do + "tag" + end + + @impl true def mapping do %{ settings: %{ @@ -48,9 +61,7 @@ defmodule Philomena.Tags.Elasticsearch do } end - # preload([ - # :aliased_tag, :aliases, :implied_tags, :implied_by_tags - # ]) + @impl true def as_json(tag) do %{ id: tag.id, diff --git a/lib/philomena/tags/tag.ex b/lib/philomena/tags/tag.ex index 8eea46dc..023f329c 100644 --- a/lib/philomena/tags/tag.ex +++ b/lib/philomena/tags/tag.ex @@ -2,11 +2,6 @@ defmodule Philomena.Tags.Tag do use Ecto.Schema import Ecto.Changeset - use Philomena.Elasticsearch, - definition: Philomena.Tags.Elasticsearch, - index_name: "tags", - doc_type: "tag" - alias Philomena.DnpEntries.DnpEntry alias Philomena.UserLinks.UserLink alias Philomena.Tags.Tag diff --git a/lib/philomena_web/controllers/activity_controller.ex b/lib/philomena_web/controllers/activity_controller.ex index 453a4dfa..c58d41c7 100644 --- a/lib/philomena_web/controllers/activity_controller.ex +++ b/lib/philomena_web/controllers/activity_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.ActivityController do use PhilomenaWeb, :controller alias PhilomenaWeb.ImageLoader + alias Philomena.Elasticsearch alias Philomena.{Images.Image, ImageFeatures.ImageFeature, Comments.Comment, Channels.Channel, Topics.Topic, Forums.Forum} alias Philomena.Interactions alias Philomena.Repo @@ -26,7 +27,8 @@ defmodule PhilomenaWeb.ActivityController do ) comments = - Comment.search_records( + Elasticsearch.search_records( + Comment, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/controllers/admin/report_controller.ex b/lib/philomena_web/controllers/admin/report_controller.ex index 4cb7f4c6..2f569a4d 100644 --- a/lib/philomena_web/controllers/admin/report_controller.ex +++ b/lib/philomena_web/controllers/admin/report_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.Admin.ReportController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.Textile.Renderer alias Philomena.Reports.Report alias Philomena.Reports.Query @@ -61,7 +62,8 @@ defmodule PhilomenaWeb.Admin.ReportController do defp load_reports(conn, query) do reports = - Report.search_records( + Elasticsearch.search_records( + Report, %{ query: query, sort: sorts() diff --git a/lib/philomena_web/controllers/api/json/search/comment_controller.ex b/lib/philomena_web/controllers/api/json/search/comment_controller.ex index d8ea3033..e021193e 100644 --- a/lib/philomena_web/controllers/api/json/search/comment_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/comment_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do use PhilomenaWeb, :controller alias PhilomenaWeb.CommentJson + alias Philomena.Elasticsearch alias Philomena.Comments.Comment alias Philomena.Comments.Query import Ecto.Query @@ -13,7 +14,8 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do case Query.compile(user, params["q"] || "") do {:ok, query} -> comments = - Comment.search_records( + Elasticsearch.search_records( + Comment, %{ query: %{ bool: %{ @@ -40,4 +42,4 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do |> json(%{error: msg}) end end -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/api/json/search/gallery_controller.ex b/lib/philomena_web/controllers/api/json/search/gallery_controller.ex index 779adc96..67b5e9bb 100644 --- a/lib/philomena_web/controllers/api/json/search/gallery_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/gallery_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do use PhilomenaWeb, :controller alias PhilomenaWeb.GalleryJson + alias Philomena.Elasticsearch alias Philomena.Galleries.Gallery alias Philomena.Galleries.Query import Ecto.Query @@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do case Query.compile(params["q"] || "") do {:ok, query} -> galleries = - Gallery.search_records( + Elasticsearch.search_records( + Gallery, %{ query: query, sort: %{created_at: :desc} @@ -27,4 +29,4 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do |> json(%{error: msg}) end end -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/api/json/search/post_controller.ex b/lib/philomena_web/controllers/api/json/search/post_controller.ex index 3292bf7b..48e47f6a 100644 --- a/lib/philomena_web/controllers/api/json/search/post_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/post_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do use PhilomenaWeb, :controller alias PhilomenaWeb.PostJson + alias Philomena.Elasticsearch alias Philomena.Posts.Post alias Philomena.Posts.Query import Ecto.Query @@ -12,7 +13,8 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do case Query.compile(user, params["q"] || "") do {:ok, query} -> posts = - Post.search_records( + Elasticsearch.search_records( + Post, %{ query: %{ bool: %{ @@ -37,4 +39,4 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do |> json(%{error: msg}) end end -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/api/json/search/tag_controller.ex b/lib/philomena_web/controllers/api/json/search/tag_controller.ex index cbb1146c..7ea6a2b7 100644 --- a/lib/philomena_web/controllers/api/json/search/tag_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/tag_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do use PhilomenaWeb, :controller alias PhilomenaWeb.TagJson + alias Philomena.Elasticsearch alias Philomena.Tags.Tag alias Philomena.Tags.Query import Ecto.Query @@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do case Query.compile(params["q"] || "") do {:ok, query} -> tags = - Tag.search_records( + Elasticsearch.search_records( + Tag, %{query: query, sort: %{images: :desc}}, conn.assigns.pagination, preload(Tag, [:aliased_tag, :aliases, :implied_tags, :implied_by_tags, :dnp_entries]) @@ -24,4 +26,4 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do |> json(%{error: msg}) end end -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/api/rss/watched_controller.ex b/lib/philomena_web/controllers/api/rss/watched_controller.ex index 916b84e6..7a9a4ae8 100644 --- a/lib/philomena_web/controllers/api/rss/watched_controller.ex +++ b/lib/philomena_web/controllers/api/rss/watched_controller.ex @@ -1,34 +1,11 @@ defmodule PhilomenaWeb.Api.Rss.WatchedController do use PhilomenaWeb, :controller - alias Philomena.Images.{Image, Query} - import Ecto.Query + alias PhilomenaWeb.ImageLoader def index(conn, _params) do - user = conn.assigns.current_user - filter = conn.assigns.compiled_filter + {:ok, {images, _tags}} = ImageLoader.search_string(conn, "my:watched") - {:ok, query} = Query.compile(user, "my:watched") - - images = - Image.search_records( - %{ - query: %{ - bool: %{ - must: query, - must_not: [ - filter, - %{term: %{hidden_from_users: true}} - ] - } - }, - sort: %{created_at: :desc} - }, - conn.assigns.image_pagination, - Image |> preload(:tags) - ) - - conn - |> render("index.rss", images: images) + render(conn, "index.rss", images: images) end -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/comment_controller.ex b/lib/philomena_web/controllers/comment_controller.ex index 1cad838b..4bb6cfb2 100644 --- a/lib/philomena_web/controllers/comment_controller.ex +++ b/lib/philomena_web/controllers/comment_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.CommentController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.{Comments.Query, Comments.Comment, Textile.Renderer} import Ecto.Query @@ -18,7 +19,8 @@ defmodule PhilomenaWeb.CommentController do defp render_index({:ok, query}, conn, user) do comments = - Comment.search_records( + Elasticsearch.search_records( + Comment, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/controllers/gallery_controller.ex b/lib/philomena_web/controllers/gallery_controller.ex index aa0f0a91..c79242a1 100644 --- a/lib/philomena_web/controllers/gallery_controller.ex +++ b/lib/philomena_web/controllers/gallery_controller.ex @@ -3,6 +3,7 @@ defmodule PhilomenaWeb.GalleryController do alias PhilomenaWeb.ImageLoader alias PhilomenaWeb.NotificationCountPlug + alias Philomena.Elasticsearch alias Philomena.ImageSorter alias Philomena.Interactions alias Philomena.Galleries.Gallery @@ -14,7 +15,8 @@ defmodule PhilomenaWeb.GalleryController do def index(conn, params) do galleries = - Gallery.search_records( + Elasticsearch.search_records( + Gallery, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/controllers/image/random_controller.ex b/lib/philomena_web/controllers/image/random_controller.ex index 8f776a46..2fe28e14 100644 --- a/lib/philomena_web/controllers/image/random_controller.ex +++ b/lib/philomena_web/controllers/image/random_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.Image.RandomController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.Images.Query alias Philomena.Images.Image alias Philomena.ImageSorter @@ -30,7 +31,8 @@ defmodule PhilomenaWeb.Image.RandomController do defp random_image_id(query, filter) do sort = ImageSorter.parse_sort(%{"sf" => "random"}) - Image.search_records( + Elasticsearch.search_records( + Image, %{ query: %{ bool: %{ @@ -51,4 +53,4 @@ defmodule PhilomenaWeb.Image.RandomController do defp unwrap([image]), do: image.id defp unwrap([]), do: nil -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/post_controller.ex b/lib/philomena_web/controllers/post_controller.ex index c8790996..cb7bc2f7 100644 --- a/lib/philomena_web/controllers/post_controller.ex +++ b/lib/philomena_web/controllers/post_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.PostController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.{Posts.Query, Posts.Post, Textile.Renderer} import Ecto.Query @@ -18,7 +19,8 @@ defmodule PhilomenaWeb.PostController do defp render_index({:ok, query}, conn, user) do posts = - Post.search_records( + Elasticsearch.search_records( + Post, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/controllers/profile_controller.ex b/lib/philomena_web/controllers/profile_controller.ex index 1977d785..60c5b2af 100644 --- a/lib/philomena_web/controllers/profile_controller.ex +++ b/lib/philomena_web/controllers/profile_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.ProfileController do use PhilomenaWeb, :controller alias PhilomenaWeb.ImageLoader + alias Philomena.Elasticsearch alias Philomena.Textile.Renderer alias Philomena.UserStatistics.UserStatistic alias Philomena.Users.User @@ -61,7 +62,8 @@ defmodule PhilomenaWeb.ProfileController do recent_artwork = recent_artwork(conn, tags) recent_comments = - Comment.search_records( + Elasticsearch.search_records( + Comment, %{ query: %{ bool: %{ @@ -88,7 +90,8 @@ defmodule PhilomenaWeb.ProfileController do |> Enum.zip(recent_comments) recent_posts = - Post.search_records( + Elasticsearch.search_records( + Post, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/controllers/stat_controller.ex b/lib/philomena_web/controllers/stat_controller.ex index 9bf1e726..de2c7600 100644 --- a/lib/philomena_web/controllers/stat_controller.ex +++ b/lib/philomena_web/controllers/stat_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.StatController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.Images.Image alias Philomena.Comments.Comment alias Philomena.Topics.Topic @@ -51,7 +52,10 @@ defmodule PhilomenaWeb.StatController do Application.get_env(:philomena, :aggregation_json) |> Jason.decode!() - {Image.search(data["images"]), Comment.search(data["comments"])} + { + Elasticsearch.search(Image, data["images"]), + Elasticsearch.search(Comment, data["comments"]) + } end defp forums do diff --git a/lib/philomena_web/controllers/tag/autocomplete_controller.ex b/lib/philomena_web/controllers/tag/autocomplete_controller.ex index 55b9cf8a..89c4356d 100644 --- a/lib/philomena_web/controllers/tag/autocomplete_controller.ex +++ b/lib/philomena_web/controllers/tag/autocomplete_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do use PhilomenaWeb, :controller + alias Philomena.Elasticsearch alias Philomena.Tags.Tag def show(conn, params) do @@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do [] term -> - Tag.search_records( + Elasticsearch.search_records( + Tag, %{ query: %{ bool: %{ @@ -38,4 +40,4 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do |> String.trim() end defp query(_params), do: nil -end \ No newline at end of file +end diff --git a/lib/philomena_web/controllers/tag_controller.ex b/lib/philomena_web/controllers/tag_controller.ex index 086bc0e1..3671df06 100644 --- a/lib/philomena_web/controllers/tag_controller.ex +++ b/lib/philomena_web/controllers/tag_controller.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.TagController do use PhilomenaWeb, :controller alias PhilomenaWeb.ImageLoader + alias Philomena.Elasticsearch alias Philomena.{Tags, Tags.Tag} alias Philomena.Textile.Renderer alias Philomena.Interactions @@ -16,7 +17,8 @@ defmodule PhilomenaWeb.TagController do with {:ok, query} <- Tags.Query.compile(query_string) do tags = - Tag.search_records( + Elasticsearch.search_records( + Tag, %{ query: query, size: 250, diff --git a/lib/philomena_web/image_json.ex b/lib/philomena_web/image_json.ex index edc86ba5..15078aa8 100644 --- a/lib/philomena_web/image_json.ex +++ b/lib/philomena_web/image_json.ex @@ -19,7 +19,7 @@ defmodule PhilomenaWeb.ImageJson do tag_ids: Enum.map(image.tags, & &1.id), uploader: if(!!image.user and !image.anonymous, do: image.user.name), uploader_id: if(!!image.user and !image.anonymous, do: image.user.id), - wilson_score: Philomena.Images.Elasticsearch.wilson_score(image), + wilson_score: Philomena.Images.ElasticsearchIndex.wilson_score(image), score: image.score, upvotes: image.upvotes_count, downvotes: image.downvotes_count, @@ -35,4 +35,4 @@ defmodule PhilomenaWeb.ImageJson do processed: image.processed } end -end \ No newline at end of file +end diff --git a/lib/philomena_web/image_loader.ex b/lib/philomena_web/image_loader.ex index 6b4125a8..f977f95f 100644 --- a/lib/philomena_web/image_loader.ex +++ b/lib/philomena_web/image_loader.ex @@ -1,4 +1,5 @@ defmodule PhilomenaWeb.ImageLoader do + alias Philomena.Elasticsearch alias Philomena.Images.{Image, Query} alias Philomena.Textile.Renderer alias Philomena.Tags.Tag @@ -27,7 +28,8 @@ defmodule PhilomenaWeb.ImageLoader do filters = create_filters(conn, user, filter) records = - Image.search_records( + Elasticsearch.search_records( + Image, %{ query: %{ bool: %{ diff --git a/lib/philomena_web/views/tag_view.ex b/lib/philomena_web/views/tag_view.ex index 4b6841dd..03540725 100644 --- a/lib/philomena_web/views/tag_view.ex +++ b/lib/philomena_web/views/tag_view.ex @@ -2,6 +2,7 @@ defmodule PhilomenaWeb.TagView do use PhilomenaWeb, :view # this is bad practice, don't copy this. + alias Philomena.Elasticsearch alias Philomena.Tags.Tag alias Philomena.Repo import Ecto.Query @@ -124,7 +125,8 @@ defmodule PhilomenaWeb.TagView do end defp implied_by_multitag(tag_names, ignore_tag_names) do - Tag.search_records( + Elasticsearch.search_records( + Tag, %{ query: %{ bool: %{