break compile time dependencies for elasticsearch index usage

lib/mix/tasks/reindex_all.ex

@@ -1,6 +1,7 @@
 defmodule Mix.Tasks.ReindexAll do
   use Mix.Task
 
+  alias Philomena.Elasticsearch
   alias Philomena.{Comments.Comment, Galleries.Gallery, Posts.Post, Images.Image, Reports.Report, Tags.Tag}
   alias Philomena.{Comments, Galleries, Posts, Images, Tags}
   alias Philomena.Polymorphic
@@ -16,21 +17,21 @@ defmodule Mix.Tasks.ReindexAll do
     {:ok, _apps} = Application.ensure_all_started(:philomena)
 
     for {context, schema} <- [{Images, Image}, {Comments, Comment}, {Galleries, Gallery}, {Tags, Tag}, {Posts, Post}] do
-      schema.delete_index!
+      Elasticsearch.delete_index!(schema)
-      schema.create_index!
+      Elasticsearch.create_index!(schema)
 
-      schema.reindex(schema |> preload(^context.indexing_preloads()))
+      Elasticsearch.reindex(preload(schema, ^context.indexing_preloads()), schema)
     end
 
     # Reports are a bit special
 
-    Report.delete_index!
+    Elasticsearch.delete_index!(Report)
-    Report.create_index!
+    Elasticsearch.create_index!(Report)
 
     Report
     |> preload([:user, :admin])
     |> Repo.all()
     |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type])
-    |> Enum.map(&Report.index_document/1)
+    |> Enum.map(&Elasticsearch.index_document(&1, Report))
   end
 end

lib/philomena/batch.ex

@@ -0,0 +1,55 @@
+defmodule Philomena.Batch do
+  alias Philomena.Repo
+  import Ecto.Query
+
+  @doc """
+  Load records from the given queryable in batches, to avoid locking.
+
+  Valid options:
+    * :batch_size
+    * :id_field
+  """
+  def record_batches(queryable, opts \\ [], callback) do
+    query_batches(queryable, opts, &callback.(Repo.all(&1)))
+  end
+
+  @doc """
+  Load queries from the given queryable in batches, to avoid locking.
+
+  Valid options:
+    * :batch_size
+    * :id_field
+  """
+  def query_batches(queryable, opts \\ [], callback) do
+    ids = load_ids(queryable, -1, opts)
+
+    query_batches(queryable, opts, callback, ids)
+  end
+
+  defp query_batches(_queryable, _opts, _callback, []), do: []
+  defp query_batches(queryable, opts, callback, ids) do
+    id_field = Keyword.get(opts, :id_field, :id)
+
+    queryable
+    |> where([m], field(m, ^id_field) in ^ids)
+    |> callback.()
+
+    ids = load_ids(queryable, Enum.max(ids), opts)
+
+    query_batches(queryable, opts, callback, ids)
+  end
+
+  defp load_ids(queryable, max_id, opts) do
+    id_field = Keyword.get(opts, :id_field, :id)
+    batch_size = Keyword.get(opts, :batch_size, 1000)
+
+    queryable
+    |> exclude(:preload)
+    |> exclude(:order_by)
+    |> order_by(asc: ^id_field)
+    |> where([m], field(m, ^id_field) > ^max_id)
+    |> select([m], field(m, ^id_field))
+    |> limit(^batch_size)
+    |> Repo.all()
+  end
+end

lib/philomena/comments.ex

@@ -7,6 +7,7 @@ defmodule Philomena.Comments do
   alias Ecto.Multi
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Reports.Report
   alias Philomena.Comments.Comment
   alias Philomena.Images.Image
@@ -194,7 +195,7 @@ defmodule Philomena.Comments do
       |> preload(^indexing_preloads())
       |> where(id: ^comment.id)
       |> Repo.one()
-      |> Comment.index_document()
+      |> Elasticsearch.index_document(Comment)
     end
 
     comment
@@ -205,7 +206,7 @@ defmodule Philomena.Comments do
       Comment
       |> preload(^indexing_preloads())
       |> where(image_id: ^image.id)
-      |> Comment.reindex()
+      |> Elasticsearch.reindex(Comment)
     end
 
     image

lib/philomena/comments/comment.ex

@@ -2,11 +2,6 @@ defmodule Philomena.Comments.Comment do
   use Ecto.Schema
   import Ecto.Changeset
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Comments.Elasticsearch,
-    index_name: "comments",
-    doc_type: "comment"
-
   alias Philomena.Images.Image
   alias Philomena.Users.User
 

lib/philomena/comments/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Comments.Elasticsearch do
+defmodule Philomena.Comments.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "comments"
+  end
+
+  @impl true
+  def doc_type do
+    "comment"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -29,6 +42,7 @@ defmodule Philomena.Comments.Elasticsearch do
     }
   end
 
+  @impl true
   def as_json(comment) do
     %{
       id: comment.id,

lib/philomena/elasticsearch.ex

@@ -1,117 +1,120 @@
 defmodule Philomena.Elasticsearch do
-  defmacro __using__(opts) do
+  alias Philomena.Batch
-    definition = Keyword.fetch!(opts, :definition)
-    index_name = Keyword.fetch!(opts, :index_name)
-    doc_type = Keyword.fetch!(opts, :doc_type)
-
-    elastic_url = Application.get_env(:philomena, :elasticsearch_url)
-
-    quote do
   alias Philomena.Repo
-      import Ecto.Query, warn: false
   require Logger
+  import Ecto.Query
+
+  alias Philomena.Comments.Comment
+  alias Philomena.Galleries.Gallery
+  alias Philomena.Images.Image
+  alias Philomena.Posts.Post
+  alias Philomena.Reports.Report
+  alias Philomena.Tags.Tag
+
+  alias Philomena.Comments.ElasticsearchIndex, as: CommentIndex
+  alias Philomena.Galleries.ElasticsearchIndex, as: GalleryIndex
+  alias Philomena.Images.ElasticsearchIndex, as: ImageIndex
+  alias Philomena.Posts.ElasticsearchIndex, as: PostIndex
+  alias Philomena.Reports.ElasticsearchIndex, as: ReportIndex
+  alias Philomena.Tags.ElasticsearchIndex, as: TagIndex
+
+  defp index_for(Comment), do: CommentIndex
+  defp index_for(Gallery), do: GalleryIndex
+  defp index_for(Image), do: ImageIndex
+  defp index_for(Post), do: PostIndex
+  defp index_for(Report), do: ReportIndex
+  defp index_for(Tag), do: TagIndex
+
+  defp elastic_url do
+    Application.get_env(:philomena, :elasticsearch_url)
+  end
+
+  def create_index!(module) do
+    index = index_for(module)
 
-      def create_index! do
     Elastix.Index.create(
-          unquote(elastic_url),
+      elastic_url(),
-          unquote(index_name),
+      index.index_name(),
-          unquote(definition).mapping()
+      index.mapping()
     )
   end
 
-      def delete_index! do
+  def delete_index!(module) do
-        Elastix.Index.delete(unquote(elastic_url), unquote(index_name))
+    index = index_for(module)
+
+    Elastix.Index.delete(elastic_url(), index.index_name())
   end
 
-      def index_document(doc) do
+  def index_document(doc, module) do
-        data = unquote(definition).as_json(doc)
+    index = index_for(module)
+    data = index.as_json(doc)
 
     Elastix.Document.index(
-          unquote(elastic_url),
+      elastic_url(),
-          unquote(index_name),
+      index.index_name(),
-          [unquote(doc_type)],
+      [index.doc_type()],
       data.id,
       data
     )
   end
 
-      def delete_document(id) do
+  def delete_document(id, module) do
+    index = index_for(module)
+
     Elastix.Document.delete(
-          unquote(elastic_url),
+      elastic_url(),
-          unquote(index_name),
+      index.index_name(),
-          unquote(doc_type),
+      index.doc_type(),
       id
     )
   end
 
-      def reindex(ecto_query, batch_size \\ 1000) do
+  def reindex(queryable, module, opts \\ []) do
-        ids =
+    index = index_for(module)
-          ecto_query
-          |> exclude(:preload)
-          |> exclude(:order_by)
-          |> order_by(asc: :id)
-          |> select([m], m.id)
-          |> limit(^batch_size)
-          |> Repo.all()
 
-        reindex(ecto_query, batch_size, ids)
+    Batch.record_batches(queryable, opts, fn records ->
-      end
-
-      def reindex(ecto_query, batch_size, []), do: nil
-
-      def reindex(ecto_query, batch_size, ids) do
       lines =
-          ecto_query
+        Enum.flat_map(records, fn record ->
-          |> where([m], m.id in ^ids)
+          doc = index.as_json(record)
-          |> Repo.all()
-          |> Enum.flat_map(fn m ->
-            doc = unquote(definition).as_json(m)
 
           [
-              %{index: %{_index: unquote(index_name), _type: unquote(doc_type), _id: doc.id}},
+            %{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}},
             doc
           ]
         end)
 
-        Elastix.Bulk.post(unquote(elastic_url), lines,
+      Elastix.Bulk.post(
-          index: unquote(index_name),
+        elastic_url(),
+        lines,
+        index: index.index_name(),
         httpoison_options: [timeout: 30_000]
       )
-
+    end)
-        ids =
-          ecto_query
-          |> exclude(:preload)
-          |> exclude(:order_by)
-          |> order_by(asc: :id)
-          |> where([m], m.id > ^Enum.max(ids))
-          |> select([m], m.id)
-          |> limit(^batch_size)
-          |> Repo.all()
-
-        reindex(ecto_query, batch_size, ids)
   end
 
-      def search(query_body) do
+  def search(module, query_body) do
+    index = index_for(module)
+
     {:ok, %{body: results, status_code: 200}} =
       Elastix.Search.search(
-            unquote(elastic_url),
+        elastic_url(),
-            unquote(index_name),
+        index.index_name(),
-            [unquote(doc_type)],
+        [index.doc_type()],
         query_body
       )
 
     results
   end
 
-      def search_results(elastic_query, pagination_params \\ %{}) do
+  def search_results(module, elastic_query, pagination_params \\ %{}) do
     page_number = pagination_params[:page_number] || 1
     page_size = pagination_params[:page_size] || 25
     elastic_query = Map.merge(elastic_query, %{from: (page_number - 1) * page_size, size: page_size, _source: false})
 
-        results = search(elastic_query)
+    results = search(module, elastic_query)
     time = results["took"]
     count = results["hits"]["total"]
-        entries = results["hits"]["hits"] |> Enum.map(&String.to_integer(&1["_id"]))
+    entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"]))
 
     Logger.debug("[Elasticsearch] Query took #{time}ms")
 
@@ -124,8 +127,8 @@ defmodule Philomena.Elasticsearch do
     }
   end
 
-      def search_records(elastic_query, pagination_params \\ %{}, ecto_query \\ __MODULE__) do
+  def search_records(module, elastic_query, pagination_params \\ %{}, ecto_query \\ __MODULE__) do
-        page = search_results(elastic_query, pagination_params)
+    page = search_results(module, elastic_query, pagination_params)
     ids = page.entries
 
     records =
@@ -137,5 +140,3 @@ defmodule Philomena.Elasticsearch do
     %{page | entries: records}
   end
 end
-  end
-end

lib/philomena/elasticsearch_index.ex

@@ -0,0 +1,18 @@
+defmodule Philomena.ElasticsearchIndex do
+  # Returns the index name for the index.
+  # This is usually a collection name like "images".
+  @callback index_name() :: String.t()
+
+  # Returns the document type for the index.
+  # This is usually the singular of the index name, like "image".
+  # 
+  # TODO: Remove for ES 7.0
+  @callback doc_type() :: String.t()
+
+  # Returns the mapping and settings for the index.
+  @callback mapping() :: map()
+
+  # Returns the JSON representation of the given struct
+  # for indexing in Elasticsearch.
+  @callback as_json(struct()) :: map()
+end

lib/philomena/galleries.ex

@@ -7,6 +7,7 @@ defmodule Philomena.Galleries do
   alias Ecto.Multi
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Galleries.Gallery
   alias Philomena.Galleries.Interaction
   alias Philomena.Notifications
@@ -127,7 +128,7 @@ defmodule Philomena.Galleries do
       |> preload(^indexing_preloads())
       |> where(id: ^gallery.id)
       |> Repo.one()
-      |> Gallery.index_document()
+      |> Elasticsearch.index_document(Gallery)
     end
 
     gallery
@@ -135,7 +136,7 @@ defmodule Philomena.Galleries do
 
   def unindex_gallery(%Gallery{} = gallery) do
     spawn fn ->
-      Gallery.delete_document(gallery.id)
+      Elasticsearch.delete_document(gallery.id, Gallery)
     end
 
     gallery

lib/philomena/galleries/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Galleries.Elasticsearch do
+defmodule Philomena.Galleries.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "galleries"
+  end
+
+  @impl true
+  def doc_type do
+    "gallery"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -28,7 +41,7 @@ defmodule Philomena.Galleries.Elasticsearch do
     }
   end
 
-  # [:subscribers, :creator, :interactions]
+  @impl true
   def as_json(gallery) do
     %{
       id: gallery.id,

lib/philomena/galleries/gallery.ex

@@ -2,11 +2,6 @@ defmodule Philomena.Galleries.Gallery do
   use Ecto.Schema
   import Ecto.Changeset
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Galleries.Elasticsearch,
-    index_name: "galleries",
-    doc_type: "gallery"
-
   alias Philomena.Images.Image
   alias Philomena.Users.User
   alias Philomena.Galleries.Interaction

lib/philomena/image_navigator.ex

@@ -1,6 +1,7 @@
 defmodule Philomena.ImageNavigator do
   alias Philomena.ImageSorter
-  alias Philomena.Images.{Image, Elasticsearch}
+  alias Philomena.Images.{Image, ElasticsearchIndex}
+  alias Philomena.Elasticsearch
   alias Philomena.Repo
   import Ecto.Query
 
@@ -30,7 +31,7 @@ defmodule Philomena.ImageNavigator do
       |> preload(:gallery_interactions)
       |> Repo.one()
       |> Map.merge(empty_fields())
-      |> Elasticsearch.as_json()
+      |> ElasticsearchIndex.as_json()
 
     sort_data = ImageSorter.parse_sort(params)
 
@@ -42,7 +43,8 @@ defmodule Philomena.ImageNavigator do
     sorts = sortify(sorts, image_index)
     filters = filterify(filters, image_index)
 
-    Image.search_records(
+    Elasticsearch.search_records(
+      Image,
       %{
         query: %{
           bool: %{

lib/philomena/images.ex

@@ -8,6 +8,7 @@ defmodule Philomena.Images do
   alias Ecto.Multi
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Images.Image
   alias Philomena.Images.Hider
   alias Philomena.Images.Uploader
@@ -483,7 +484,7 @@ defmodule Philomena.Images do
       Image
       |> preload(^indexing_preloads())
       |> where([i], i.id in ^image_ids)
-      |> Image.reindex()
+      |> Elasticsearch.reindex(Image)
     end
 
     image_ids

lib/philomena/images/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Images.Elasticsearch do
+defmodule Philomena.Images.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "images"
+  end
+
+  @impl true
+  def doc_type do
+    "image"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -77,10 +90,7 @@ defmodule Philomena.Images.Elasticsearch do
     }
   end
 
-  # preload([
+  @impl true
-  #   :user, :deleter, :gallery_interactions, :upvoters, :downvoters, :favers,
-  #   :hiders, tags: [:aliases, :aliased_tag]
-  # ])
   def as_json(image) do
     %{
       id: image.id,

lib/philomena/images/image.ex

@@ -1,11 +1,6 @@
 defmodule Philomena.Images.Image do
   use Ecto.Schema
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Images.Elasticsearch,
-    index_name: "images",
-    doc_type: "image"
-
   import Ecto.Changeset
   import Ecto.Query
 

lib/philomena/posts.ex

@@ -7,6 +7,7 @@ defmodule Philomena.Posts do
   alias Ecto.Multi
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Topics.Topic
   alias Philomena.Topics
   alias Philomena.Posts.Post
@@ -207,7 +208,7 @@ defmodule Philomena.Posts do
       |> preload(^indexing_preloads())
       |> where(id: ^post.id)
       |> Repo.one()
-      |> Post.index_document()
+      |> Elasticsearch.index_document(Post)
     end
 
     post

lib/philomena/posts/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Posts.Elasticsearch do
+defmodule Philomena.Posts.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "posts"
+  end
+
+  @impl true
+  def doc_type do
+    "post"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -36,7 +49,7 @@ defmodule Philomena.Posts.Elasticsearch do
     }
   end
 
-  # [:user, topic: :forum]
+  @impl true
   def as_json(post) do
     %{
       id: post.id,

lib/philomena/posts/post.ex

@@ -2,11 +2,6 @@ defmodule Philomena.Posts.Post do
   use Ecto.Schema
   import Ecto.Changeset
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Posts.Elasticsearch,
-    index_name: "posts",
-    doc_type: "post"
-
   alias Philomena.Users.User
   alias Philomena.Topics.Topic
 

lib/philomena/reports.ex

@@ -6,6 +6,7 @@ defmodule Philomena.Reports do
   import Ecto.Query, warn: false
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Reports.Report
   alias Philomena.Polymorphic
 
@@ -128,7 +129,7 @@ defmodule Philomena.Reports do
       |> preload([:user, :admin])
       |> Repo.all()
       |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type])
-      |> Enum.map(&Report.index_document/1)
+      |> Enum.map(&Elasticsearch.index_document(&1, Report))
     end
 
     report_ids

lib/philomena/reports/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Reports.Elasticsearch do
+defmodule Philomena.Reports.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "reports"
+  end
+
+  @impl true
+  def doc_type do
+    "report"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -32,6 +45,7 @@ defmodule Philomena.Reports.Elasticsearch do
     }
   end
 
+  @impl true
   def as_json(report) do
     %{
       id: report.id,

lib/philomena/reports/report.ex

@@ -2,11 +2,6 @@ defmodule Philomena.Reports.Report do
   use Ecto.Schema
   import Ecto.Changeset
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Reports.Elasticsearch,
-    index_name: "reports",
-    doc_type: "report"
-
   alias Philomena.Users.User
 
   schema "reports" do

lib/philomena/tags.ex

@@ -7,6 +7,7 @@ defmodule Philomena.Tags do
   alias Ecto.Multi
   alias Philomena.Repo
 
+  alias Philomena.Elasticsearch
   alias Philomena.Tags.Tag
   alias Philomena.Tags.Uploader
   alias Philomena.Images
@@ -165,12 +166,12 @@ defmodule Philomena.Tags do
 
     {:ok, tag} = Repo.delete(tag)
 
-    Tag.delete_document(tag.id)
+    Elasticsearch.delete_document(tag.id, Tag)
 
     Image
     |> where([i], i.id in ^image_ids)
     |> preload(^Images.indexing_preloads())
-    |> Image.reindex()
+    |> Elasticsearch.reindex(Image)
   end
 
   def alias_tag(%Tag{} = tag, attrs) do
@@ -234,7 +235,7 @@ defmodule Philomena.Tags do
     |> join(:inner, [i], _ in assoc(i, :tags))
     |> where([_i, t], t.id == ^tag.id)
     |> preload(^Images.indexing_preloads())
-    |> Image.reindex()
+    |> Elasticsearch.reindex(Image)
   end
 
   def unalias_tag(%Tag{} = tag) do
@@ -316,7 +317,7 @@ defmodule Philomena.Tags do
       Tag
       |> preload(^indexing_preloads())
       |> where([t], t.id in ^ids)
-      |> Tag.reindex()
+      |> Elasticsearch.reindex(Tag)
     end
 
     tags

lib/philomena/tags/elasticsearch_index.ex

@@ -1,4 +1,17 @@
-defmodule Philomena.Tags.Elasticsearch do
+defmodule Philomena.Tags.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "tags"
+  end
+
+  @impl true
+  def doc_type do
+    "tag"
+  end
+
+  @impl true
   def mapping do
     %{
       settings: %{
@@ -48,9 +61,7 @@ defmodule Philomena.Tags.Elasticsearch do
     }
   end
 
-  # preload([
+  @impl true
-  #   :aliased_tag, :aliases, :implied_tags, :implied_by_tags
-  # ])
   def as_json(tag) do
     %{
       id: tag.id,

lib/philomena/tags/tag.ex

@@ -2,11 +2,6 @@ defmodule Philomena.Tags.Tag do
   use Ecto.Schema
   import Ecto.Changeset
 
-  use Philomena.Elasticsearch,
-    definition: Philomena.Tags.Elasticsearch,
-    index_name: "tags",
-    doc_type: "tag"
-
   alias Philomena.DnpEntries.DnpEntry
   alias Philomena.UserLinks.UserLink
   alias Philomena.Tags.Tag

lib/philomena_web/controllers/activity_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.ActivityController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.ImageLoader
+  alias Philomena.Elasticsearch
   alias Philomena.{Images.Image, ImageFeatures.ImageFeature, Comments.Comment, Channels.Channel, Topics.Topic, Forums.Forum}
   alias Philomena.Interactions
   alias Philomena.Repo
@@ -26,7 +27,8 @@ defmodule PhilomenaWeb.ActivityController do
       )
 
     comments =
-      Comment.search_records(
+      Elasticsearch.search_records(
+        Comment,
         %{
           query: %{
             bool: %{

lib/philomena_web/controllers/admin/report_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.Admin.ReportController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.Textile.Renderer
   alias Philomena.Reports.Report
   alias Philomena.Reports.Query
@@ -61,7 +62,8 @@ defmodule PhilomenaWeb.Admin.ReportController do
 
   defp load_reports(conn, query) do
     reports =
-      Report.search_records(
+      Elasticsearch.search_records(
+        Report,
         %{
           query: query,
           sort: sorts()

lib/philomena_web/controllers/api/json/search/comment_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.CommentJson
+  alias Philomena.Elasticsearch
   alias Philomena.Comments.Comment
   alias Philomena.Comments.Query
   import Ecto.Query
@@ -13,7 +14,8 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do
     case Query.compile(user, params["q"] || "") do
       {:ok, query} ->
         comments =
-          Comment.search_records(
+          Elasticsearch.search_records(
+            Comment,
             %{
               query: %{
                 bool: %{

lib/philomena_web/controllers/api/json/search/gallery_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.GalleryJson
+  alias Philomena.Elasticsearch
   alias Philomena.Galleries.Gallery
   alias Philomena.Galleries.Query
   import Ecto.Query
@@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do
     case Query.compile(params["q"] || "") do
       {:ok, query} ->
         galleries =
-          Gallery.search_records(
+          Elasticsearch.search_records(
+            Gallery,
             %{
               query: query,
               sort: %{created_at: :desc}

lib/philomena_web/controllers/api/json/search/post_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.PostJson
+  alias Philomena.Elasticsearch
   alias Philomena.Posts.Post
   alias Philomena.Posts.Query
   import Ecto.Query
@@ -12,7 +13,8 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do
     case Query.compile(user, params["q"] || "") do
       {:ok, query} ->
         posts =
-          Post.search_records(
+          Elasticsearch.search_records(
+            Post,
             %{
               query: %{
                 bool: %{

lib/philomena_web/controllers/api/json/search/tag_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.TagJson
+  alias Philomena.Elasticsearch
   alias Philomena.Tags.Tag
   alias Philomena.Tags.Query
   import Ecto.Query
@@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do
     case Query.compile(params["q"] || "") do
       {:ok, query} ->
         tags =
-          Tag.search_records(
+          Elasticsearch.search_records(
+            Tag,
             %{query: query, sort: %{images: :desc}},
             conn.assigns.pagination,
             preload(Tag, [:aliased_tag, :aliases, :implied_tags, :implied_by_tags, :dnp_entries])

lib/philomena_web/controllers/api/rss/watched_controller.ex

@@ -1,34 +1,11 @@
 defmodule PhilomenaWeb.Api.Rss.WatchedController do
   use PhilomenaWeb, :controller
 
-  alias Philomena.Images.{Image, Query}
+  alias PhilomenaWeb.ImageLoader
-  import Ecto.Query
 
   def index(conn, _params) do
-    user = conn.assigns.current_user
+    {:ok, {images, _tags}} = ImageLoader.search_string(conn, "my:watched")
-    filter = conn.assigns.compiled_filter
 
-    {:ok, query} = Query.compile(user, "my:watched")
+    render(conn, "index.rss", images: images)
-
-    images =
-      Image.search_records(
-        %{
-          query: %{
-            bool: %{
-              must: query,
-              must_not: [
-                filter,
-                %{term: %{hidden_from_users: true}}
-              ]
-            }
-          },
-          sort: %{created_at: :desc}
-        },
-        conn.assigns.image_pagination,
-        Image |> preload(:tags)
-      )
-
-    conn
-    |> render("index.rss", images: images)
   end
 end

lib/philomena_web/controllers/comment_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.CommentController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.{Comments.Query, Comments.Comment, Textile.Renderer}
   import Ecto.Query
 
@@ -18,7 +19,8 @@ defmodule PhilomenaWeb.CommentController do
 
   defp render_index({:ok, query}, conn, user) do
     comments =
-      Comment.search_records(
+      Elasticsearch.search_records(
+        Comment,
         %{
           query: %{
             bool: %{

lib/philomena_web/controllers/gallery_controller.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaWeb.GalleryController do
 
   alias PhilomenaWeb.ImageLoader
   alias PhilomenaWeb.NotificationCountPlug
+  alias Philomena.Elasticsearch
   alias Philomena.ImageSorter
   alias Philomena.Interactions
   alias Philomena.Galleries.Gallery
@@ -14,7 +15,8 @@ defmodule PhilomenaWeb.GalleryController do
 
   def index(conn, params) do
     galleries =
-      Gallery.search_records(
+      Elasticsearch.search_records(
+        Gallery,
         %{
           query: %{
             bool: %{

lib/philomena_web/controllers/image/random_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.Image.RandomController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.Images.Query
   alias Philomena.Images.Image
   alias Philomena.ImageSorter
@@ -30,7 +31,8 @@ defmodule PhilomenaWeb.Image.RandomController do
   defp random_image_id(query, filter) do
     sort = ImageSorter.parse_sort(%{"sf" => "random"})
 
-    Image.search_records(
+    Elasticsearch.search_records(
+      Image,
       %{
         query: %{
           bool: %{

lib/philomena_web/controllers/post_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.PostController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.{Posts.Query, Posts.Post, Textile.Renderer}
   import Ecto.Query
 
@@ -18,7 +19,8 @@ defmodule PhilomenaWeb.PostController do
 
   defp render_index({:ok, query}, conn, user) do
     posts =
-      Post.search_records(
+      Elasticsearch.search_records(
+        Post,
         %{
           query: %{
             bool: %{

lib/philomena_web/controllers/profile_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.ProfileController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.ImageLoader
+  alias Philomena.Elasticsearch
   alias Philomena.Textile.Renderer
   alias Philomena.UserStatistics.UserStatistic
   alias Philomena.Users.User
@@ -61,7 +62,8 @@ defmodule PhilomenaWeb.ProfileController do
     recent_artwork = recent_artwork(conn, tags)
 
     recent_comments =
-      Comment.search_records(
+      Elasticsearch.search_records(
+        Comment,
         %{
           query: %{
             bool: %{
@@ -88,7 +90,8 @@ defmodule PhilomenaWeb.ProfileController do
       |> Enum.zip(recent_comments)
 
     recent_posts =
-      Post.search_records(
+      Elasticsearch.search_records(
+        Post,
         %{
           query: %{
             bool: %{

lib/philomena_web/controllers/stat_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.StatController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.Images.Image
   alias Philomena.Comments.Comment
   alias Philomena.Topics.Topic
@@ -51,7 +52,10 @@ defmodule PhilomenaWeb.StatController do
       Application.get_env(:philomena, :aggregation_json)
       |> Jason.decode!()
 
-    {Image.search(data["images"]), Comment.search(data["comments"])}
+    {
+      Elasticsearch.search(Image, data["images"]),
+      Elasticsearch.search(Comment, data["comments"])
+    }
   end
 
   defp forums do

lib/philomena_web/controllers/tag/autocomplete_controller.ex

@@ -1,6 +1,7 @@
 defmodule PhilomenaWeb.Tag.AutocompleteController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Elasticsearch
   alias Philomena.Tags.Tag
 
   def show(conn, params) do
@@ -10,7 +11,8 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do
           []
 
         term ->
-          Tag.search_records(
+          Elasticsearch.search_records(
+            Tag,
             %{
               query: %{
                 bool: %{

lib/philomena_web/controllers/tag_controller.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.TagController do
   use PhilomenaWeb, :controller
 
   alias PhilomenaWeb.ImageLoader
+  alias Philomena.Elasticsearch
   alias Philomena.{Tags, Tags.Tag}
   alias Philomena.Textile.Renderer
   alias Philomena.Interactions
@@ -16,7 +17,8 @@ defmodule PhilomenaWeb.TagController do
 
     with {:ok, query} <- Tags.Query.compile(query_string) do
       tags =
-        Tag.search_records(
+        Elasticsearch.search_records(
+          Tag,
           %{
             query: query,
             size: 250,

lib/philomena_web/image_json.ex

@@ -19,7 +19,7 @@ defmodule PhilomenaWeb.ImageJson do
       tag_ids: Enum.map(image.tags, & &1.id),
       uploader: if(!!image.user and !image.anonymous, do: image.user.name),
       uploader_id: if(!!image.user and !image.anonymous, do: image.user.id),
-      wilson_score: Philomena.Images.Elasticsearch.wilson_score(image),
+      wilson_score: Philomena.Images.ElasticsearchIndex.wilson_score(image),
       score: image.score,
       upvotes: image.upvotes_count,
       downvotes: image.downvotes_count,

lib/philomena_web/image_loader.ex

@@ -1,4 +1,5 @@
 defmodule PhilomenaWeb.ImageLoader do
+  alias Philomena.Elasticsearch
   alias Philomena.Images.{Image, Query}
   alias Philomena.Textile.Renderer
   alias Philomena.Tags.Tag
@@ -27,7 +28,8 @@ defmodule PhilomenaWeb.ImageLoader do
     filters = create_filters(conn, user, filter)
 
     records =
-      Image.search_records(
+      Elasticsearch.search_records(
+        Image,
         %{
           query: %{
             bool: %{

lib/philomena_web/views/tag_view.ex

@@ -2,6 +2,7 @@ defmodule PhilomenaWeb.TagView do
   use PhilomenaWeb, :view
 
   # this is bad practice, don't copy this.
+  alias Philomena.Elasticsearch
   alias Philomena.Tags.Tag
   alias Philomena.Repo
   import Ecto.Query
@@ -124,7 +125,8 @@ defmodule PhilomenaWeb.TagView do
   end
 
   defp implied_by_multitag(tag_names, ignore_tag_names) do
-    Tag.search_records(
+    Elasticsearch.search_records(
+      Tag,
       %{
         query: %{
           bool: %{