Public filter search (#88)

index/all.mk

@@ -1,4 +1,4 @@
-all: comments galleries images posts reports tags
+all: comments galleries images posts reports tags filters
 
 comments:
 	$(MAKE) -f comments.mk
@@ -18,5 +18,8 @@ reports:
 tags:
 	$(MAKE) -f tags.mk
 
+filters:
+	$(MAKE) -f filters.mk
+
 clean:
 	rm -f ./*.jsonl

index/filters.mk

@@ -0,0 +1,50 @@
+DATABASE ?= philomena
+ELASTICSEARCH_URL ?= http://localhost:9200/
+ELASTICDUMP ?= elasticdump
+# uncomment if getting "redirection unexpected" error on dump_jsonl
+#SHELL=/bin/bash
+
+.ONESHELL:
+
+all: import_es
+
+import_es: dump_jsonl
+	$(ELASTICDUMP) --input=filters.jsonl --output=$(ELASTICSEARCH_URL) --output-index=filters --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id"
+
+dump_jsonl: metadata creators
+	psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_filters.jsonb_object_agg(object) from temp_filters.filter_search_json group by filter_id) to stdout;' > filters.jsonl
+	psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'drop schema temp_filters cascade;'
+	sed -i filters.jsonl -e 's/\\\\/\\/g'
+
+metadata: filter_search_json
+	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
+		insert into temp_filters.filter_search_json (filter_id, object) select f.id, jsonb_build_object(
+			'id', f.id,
+			'created_at', f.created_at,
+			'user_id', f.user_id,
+			'public', f.public,
+			'system', f.system,
+			'name', lower(f.name),
+			'description', f.description,
+			'spoilered_count', array_length(f.spoilered_tag_ids, 1),
+			'hidden_count', array_length(f.hidden_tag_ids, 1),
+			'spoilered_tag_ids', f.spoilered_tag_ids,
+			'hidden_tag_ids', f.hidden_tag_ids,
+			'spoilered_complex_str', lower(f.spoilered_complex_str),
+			'hidden_complex_str', lower(f.hidden_complex_str),
+			'user_count', f.user_count
+		) from filters f;
+	SQL
+
+creators: filter_search_json
+	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
+		insert into temp_filters.filter_search_json (filter_id, object) select f.id, jsonb_build_object('creator', lower(u.name)) from filters f left join users u on f.user_id=u.id;
+	SQL
+
+filter_search_json:
+	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
+		drop schema if exists temp_filters cascade;
+		create schema temp_filters;
+		create unlogged table temp_filters.filter_search_json (filter_id bigint not null, object jsonb not null);
+		create or replace aggregate temp_filters.jsonb_object_agg(jsonb) (sfunc = 'jsonb_concat', stype = jsonb, initcond='{}');
+	SQL

lib/mix/tasks/reindex_all.ex

@@ -9,10 +9,11 @@ defmodule Mix.Tasks.ReindexAll do
     Posts.Post,
     Images.Image,
     Reports.Report,
-    Tags.Tag
+    Tags.Tag,
+    Filters.Filter
   }
 
-  alias Philomena.{Comments, Galleries, Posts, Images, Tags}
+  alias Philomena.{Comments, Galleries, Posts, Images, Tags, Filters}
   alias Philomena.Polymorphic
   alias Philomena.Repo
   import Ecto.Query
@@ -30,7 +31,8 @@ defmodule Mix.Tasks.ReindexAll do
           {Comments, Comment},
           {Galleries, Gallery},
           {Tags, Tag},
-          {Posts, Post}
+          {Posts, Post},
+          {Filters, Filter}
         ] do
       Elasticsearch.delete_index!(schema)
       Elasticsearch.create_index!(schema)

lib/philomena/elasticsearch.ex

@@ -11,6 +11,7 @@ defmodule Philomena.Elasticsearch do
   alias Philomena.Posts.Post
   alias Philomena.Reports.Report
   alias Philomena.Tags.Tag
+  alias Philomena.Filters.Filter
 
   alias Philomena.Comments.ElasticsearchIndex, as: CommentIndex
   alias Philomena.Galleries.ElasticsearchIndex, as: GalleryIndex
@@ -18,6 +19,7 @@ defmodule Philomena.Elasticsearch do
   alias Philomena.Posts.ElasticsearchIndex, as: PostIndex
   alias Philomena.Reports.ElasticsearchIndex, as: ReportIndex
   alias Philomena.Tags.ElasticsearchIndex, as: TagIndex
+  alias Philomena.Filters.ElasticsearchIndex, as: FilterIndex
 
   defp index_for(Comment), do: CommentIndex
   defp index_for(Gallery), do: GalleryIndex
@@ -25,6 +27,7 @@ defmodule Philomena.Elasticsearch do
   defp index_for(Post), do: PostIndex
   defp index_for(Report), do: ReportIndex
   defp index_for(Tag), do: TagIndex
+  defp index_for(Filter), do: FilterIndex
 
   defp elastic_url do
     Application.get_env(:philomena, :elasticsearch_url)

lib/philomena/filters.ex

@@ -7,6 +7,9 @@ defmodule Philomena.Filters do
   alias Philomena.Repo
 
   alias Philomena.Filters.Filter
+  alias Philomena.Elasticsearch
+  alias Philomena.Filters.ElasticsearchIndex, as: FilterIndex
+  alias Philomena.IndexWorker
 
   @doc """
   Returns the list of filters.
@@ -68,6 +71,7 @@ defmodule Philomena.Filters do
     %Filter{user_id: user.id}
     |> Filter.creation_changeset(attrs)
     |> Repo.insert()
+    |> reindex_after_update()
   end
 
   @doc """
@@ -86,12 +90,14 @@ defmodule Philomena.Filters do
     filter
     |> Filter.update_changeset(attrs)
     |> Repo.update()
+    |> reindex_after_update()
   end
 
   def make_filter_public(%Filter{} = filter) do
     filter
     |> Filter.public_changeset()
     |> Repo.update()
+    |> reindex_after_update()
   end
 
   @doc """
@@ -110,6 +116,15 @@ defmodule Philomena.Filters do
     filter
     |> Filter.deletion_changeset()
     |> Repo.delete()
+    |> case do
+      {:ok, filter} ->
+        unindex_filter(filter)
+
+        {:ok, filter}
+
+      error ->
+        error
+    end
   end
 
   @doc """
@@ -162,6 +177,7 @@ defmodule Philomena.Filters do
     filter
     |> Filter.hidden_tags_changeset(hidden_tag_ids)
     |> Repo.update()
+    |> reindex_after_update()
   end
 
   def unhide_tag(filter, tag) do
@@ -170,6 +186,7 @@ defmodule Philomena.Filters do
     filter
     |> Filter.hidden_tags_changeset(hidden_tag_ids)
     |> Repo.update()
+    |> reindex_after_update()
   end
 
   def spoiler_tag(filter, tag) do
@@ -178,6 +195,7 @@ defmodule Philomena.Filters do
     filter
     |> Filter.spoilered_tags_changeset(spoilered_tag_ids)
     |> Repo.update()
+    |> reindex_after_update()
   end
 
   def unspoiler_tag(filter, tag) do
@@ -186,5 +204,45 @@ defmodule Philomena.Filters do
     filter
     |> Filter.spoilered_tags_changeset(spoilered_tag_ids)
     |> Repo.update()
+    |> reindex_after_update()
+  end
+
+  defp reindex_after_update({:ok, filter}) do
+    reindex_filter(filter)
+
+    {:ok, filter}
+  end
+
+  defp reindex_after_update(error) do
+    error
+  end
+
+  def user_name_reindex(old_name, new_name) do
+    data = FilterIndex.user_name_update_by_query(old_name, new_name)
+
+    Elasticsearch.update_by_query(Filter, data.query, data.set_replacements, data.replacements)
+  end
+
+  def reindex_filter(%Filter{} = filter) do
+    Exq.enqueue(Exq, "indexing", IndexWorker, ["Filters", "id", [filter.id]])
+
+    filter
+  end
+
+  def unindex_filter(%Filter{} = filter) do
+    Elasticsearch.delete_document(filter.id, Filter)
+
+    filter
+  end
+
+  def indexing_preloads do
+    [:user]
+  end
+
+  def perform_reindex(column, condition) do
+    Filter
+    |> preload(^indexing_preloads())
+    |> where([f], field(f, ^column) in ^condition)
+    |> Elasticsearch.reindex(Filter)
   end
 end

lib/philomena/filters/elasticsearch_index.ex

@@ -0,0 +1,73 @@
+defmodule Philomena.Filters.ElasticsearchIndex do
+  @behaviour Philomena.ElasticsearchIndex
+
+  @impl true
+  def index_name do
+    "filters"
+  end
+
+  @impl true
+  def mapping do
+    %{
+      settings: %{
+        index: %{
+          number_of_shards: 5,
+          max_result_window: 10_000_000
+        }
+      },
+      mappings: %{
+        dynamic: false,
+        properties: %{
+          id: %{type: "integer"},
+          created_at: %{type: "date"},
+          user_id: %{type: "keyword"},
+          creator: %{type: "keyword"},
+          public: %{type: "boolean"},
+          system: %{type: "boolean"},
+          name: %{type: "keyword"},
+          description: %{type: "text", analyzer: "snowball"},
+          spoilered_count: %{type: "integer"},
+          hidden_count: %{type: "integer"},
+          spoilered_tag_ids: %{type: "keyword"},
+          hidden_tag_ids: %{type: "keyword"},
+          spoilered_tags: %{type: "keyword"},
+          hidden_tags: %{type: "keyword"},
+          spoilered_complex_str: %{type: "keyword"},
+          hidden_complex_str: %{type: "keyword"},
+          user_count: %{type: "integer"}
+        }
+      }
+    }
+  end
+
+  @impl true
+  def as_json(filter) do
+    %{
+      id: filter.id,
+      created_at: filter.created_at,
+      user_id: filter.user_id,
+      creator: if(!!filter.user, do: String.downcase(filter.user.name)),
+      public: filter.public || filter.system,
+      system: filter.system,
+      name: filter.name |> String.downcase(),
+      description: filter.description,
+      spoilered_count: length(filter.spoilered_tag_ids),
+      hidden_count: length(filter.hidden_tag_ids),
+      spoilered_tag_ids: filter.spoilered_tag_ids,
+      hidden_tag_ids: filter.hidden_tag_ids,
+      spoilered_complex_str:
+        if(!!filter.spoilered_complex_str, do: String.downcase(filter.spoilered_complex_str)),
+      hidden_complex_str:
+        if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)),
+      user_count: filter.user_count
+    }
+  end
+
+  def user_name_update_by_query(old_name, new_name) do
+    %{
+      query: %{term: %{creator: old_name}},
+      replacements: [%{path: ["creator"], old: old_name, new: new_name}],
+      set_replacements: []
+    }
+  end
+end

lib/philomena/filters/query.ex

@@ -0,0 +1,47 @@
+defmodule Philomena.Filters.Query do
+  alias Philomena.Search.Parser
+
+  defp user_my_transform(%{user: %{id: id}}, "filters"),
+    do: {:ok, %{term: %{user_id: id}}}
+
+  defp user_my_transform(_ctx, _value),
+    do: {:error, "Unknown `my' value."}
+
+  defp anonymous_fields do
+    [
+      int_fields: ~W(id spoilered_count hidden_count),
+      date_fields: ~W(created_at),
+      ngram_fields: ~W(description),
+      literal_fields: ~W(name creator user_id),
+      bool_fields: ~W(public system),
+      default_field: {"name", :term}
+    ]
+  end
+
+  defp user_fields do
+    fields = anonymous_fields()
+
+    Keyword.merge(fields,
+      custom_fields: ~W(my),
+      transforms: %{"my" => &user_my_transform/2}
+    )
+  end
+
+  defp parse(fields, context, query_string) do
+    fields
+    |> Parser.parser()
+    |> Parser.parse(query_string, context)
+  end
+
+  def compile(user, query_string) do
+    query_string = query_string || ""
+
+    case user do
+      nil ->
+        parse(anonymous_fields(), %{user: nil}, query_string)
+
+      user ->
+        parse(user_fields(), %{user: user}, query_string)
+    end
+  end
+end

lib/philomena/tags.ex

@@ -17,6 +17,7 @@ defmodule Philomena.Tags do
   alias Philomena.Images
   alias Philomena.Images.Image
   alias Philomena.Users.User
+  alias Philomena.Filters
   alias Philomena.Filters.Filter
   alias Philomena.Images.Tagging
   alias Philomena.ArtistLinks.ArtistLink
@@ -301,6 +302,12 @@ defmodule Philomena.Tags do
     |> where([_i, t], t.id == ^tag.id)
     |> preload(^Images.indexing_preloads())
     |> Elasticsearch.reindex(Image)
+
+    Filter
+    |> where([f], fragment("? @> ARRAY[?]::integer[]", f.hidden_tag_ids, ^tag.id))
+    |> or_where([f], fragment("? @> ARRAY[?]::integer[]", f.spoilered_tag_ids, ^tag.id))
+    |> preload(^Filters.indexing_preloads())
+    |> Elasticsearch.reindex(Filter)
   end
 
   def unalias_tag(%Tag{} = tag) do

lib/philomena/users.ex

@@ -17,6 +17,7 @@ defmodule Philomena.Users do
   alias Philomena.Posts
   alias Philomena.Galleries
   alias Philomena.Reports
+  alias Philomena.Filters
   alias Philomena.UserRenameWorker
 
   ## Database getters
@@ -620,6 +621,7 @@ defmodule Philomena.Users do
     Posts.user_name_reindex(old_name, new_name)
     Galleries.user_name_reindex(old_name, new_name)
     Reports.user_name_reindex(old_name, new_name)
+    Filters.user_name_reindex(old_name, new_name)
   end
 
   def reactivate_user(%User{} = user) do

lib/philomena/workers/index_worker.ex

@@ -5,7 +5,8 @@ defmodule Philomena.IndexWorker do
     "Images" => Philomena.Images,
     "Posts" => Philomena.Posts,
     "Reports" => Philomena.Reports,
-    "Tags" => Philomena.Tags
+    "Tags" => Philomena.Tags,
+    "Filters" => Philomena.Filters
   }
 
   # Perform the queued index. Context function looks like the following:

lib/philomena_web/controllers/api/json/search/filter_controller.ex

@@ -0,0 +1,61 @@
+defmodule PhilomenaWeb.Api.Json.Search.FilterController do
+  use PhilomenaWeb, :controller
+
+  alias Philomena.Elasticsearch
+  alias Philomena.Filters.Filter
+  alias Philomena.Filters.Query
+  import Ecto.Query
+
+  def index(conn, params) do
+    user = conn.assigns.current_user
+
+    case Query.compile(user, params["q"] || "") do
+      {:ok, query} ->
+        filters =
+          Filter
+          |> Elasticsearch.search_definition(
+            %{
+              query: %{
+                bool: %{
+                  must: [
+                    query,
+                    %{
+                      bool: %{
+                        should:
+                          [%{term: %{public: true}}, %{term: %{system: true}}] ++
+                            user_should(user)
+                      }
+                    }
+                  ]
+                }
+              },
+              sort: [
+                %{name: :asc},
+                %{id: :desc}
+              ]
+            },
+            conn.assigns.pagination
+          )
+          |> Elasticsearch.search_records(preload(Filter, [:user]))
+
+        conn
+        |> put_view(PhilomenaWeb.Api.Json.FilterView)
+        |> render("index.json", filters: filters, total: filters.total_entries)
+
+      {:error, msg} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: msg})
+    end
+  end
+
+  defp user_should(user) do
+    case user do
+      nil ->
+        []
+
+      user ->
+        [%{term: %{user_id: user.id}}]
+    end
+  end
+end

lib/philomena_web/controllers/filter_controller.ex

@@ -1,7 +1,8 @@
 defmodule PhilomenaWeb.FilterController do
   use PhilomenaWeb, :controller
 
-  alias Philomena.{Filters, Filters.Filter, Tags.Tag}
+  alias Philomena.{Filters, Filters.Filter, Filters.Query, Tags.Tag}
+  alias Philomena.Elasticsearch
   alias Philomena.Schema.TagList
   alias Philomena.Repo
   import Ecto.Query
@@ -9,6 +10,14 @@ defmodule PhilomenaWeb.FilterController do
   plug :load_and_authorize_resource, model: Filter, except: [:index], preload: :user
   plug PhilomenaWeb.RequireUserPlug when action not in [:index, :show]
 
+  def index(conn, %{"fq" => fq}) do
+    user = conn.assigns.current_user
+
+    user
+    |> Query.compile(fq)
+    |> render_index(conn, user)
+  end
+
   def index(conn, _params) do
     user = conn.assigns.current_user
 
@@ -35,6 +44,51 @@ defmodule PhilomenaWeb.FilterController do
     )
   end
 
+  defp render_index({:ok, query}, conn, user) do
+    filters =
+      Filter
+      |> Elasticsearch.search_definition(
+        %{
+          query: %{
+            bool: %{
+              must: [query | filters(user)]
+            }
+          },
+          sort: [
+            %{name: :asc},
+            %{id: :desc}
+          ]
+        },
+        conn.assigns.pagination
+      )
+      |> Elasticsearch.search_records(preload(Filter, [:user]))
+
+    render(conn, "index.html", title: "Filters", filters: filters)
+  end
+
+  defp render_index({:error, msg}, conn, _user) do
+    render(conn, "index.html", title: "Filters", error: msg, filters: [])
+  end
+
+  defp filters(user),
+    do: [%{bool: %{should: shoulds(user)}}]
+
+  defp shoulds(user) do
+    case user do
+      nil ->
+        anonymous_should()
+
+      user ->
+        user_should(user)
+    end
+  end
+
+  defp user_should(user),
+    do: anonymous_should() ++ [%{term: %{user_id: user.id}}]
+
+  defp anonymous_should(),
+    do: [%{term: %{public: true}}, %{term: %{system: true}}]
+
   def show(conn, _params) do
     filter = conn.assigns.filter
 

lib/philomena_web/router.ex

@@ -122,6 +122,7 @@ defmodule PhilomenaWeb.Router do
       resources "/posts", PostController, only: [:index]
       resources "/comments", CommentController, only: [:index]
       resources "/galleries", GalleryController, only: [:index]
+      resources "/filters", FilterController, only: [:index]
     end
 
     # Convenience alias

lib/philomena_web/templates/filter/index.html.slime

@@ -24,6 +24,7 @@
   p
     ' By default all the filters you create are private and only visible by you. You can have as many as you like and switch between them instantly with no limits. You can also create a public filter, which can be seen and used by any user on the site, allowing you to share useful filters with others.
 
+  = if !@conn.params["fq"] do
     h2 My Filters
     = if @current_user do
       p
@@ -37,3 +38,137 @@
     h2 Global Filters
     = for filter <- @system_filters do
       = render PhilomenaWeb.FilterView, "_filter.html", conn: @conn, filter: filter
+
+  h2 Search Filters
+  p
+    ' Some users maintain custom filters which are publicly shared; you can search these filters with the box below.
+  = form_for :filters, Routes.filter_path(@conn, :index), [method: "get", class: "hform", enforce_utf8: false], fn f ->
+    .field
+      = text_input f, :fq, name: :fq, value: @conn.params["fq"], class: "input hform__text", placeholder: "Search filters", autocapitalize: "none"
+      = submit "Search", class: "hform__button button"
+
+  .fieldlabel
+    ' For more information, see the
+    a href="/pages/search_syntax" search syntax documentation
+    ' . Search results are sorted by creation date.
+
+  = if @conn.params["fq"] do
+    h2 Search Results
+    = cond do
+      - Enum.any?(@filters) ->
+        - route = fn p -> Routes.filter_path(@conn, :index, p) end
+        - pagination = render PhilomenaWeb.PaginationView, "_pagination.html", page: @filters, route: route, params: [fq: @conn.params["fq"]], conn: @conn
+
+        = for filter <- @filters do
+          = render PhilomenaWeb.FilterView, "_filter.html", conn: @conn, filter: filter
+
+        .block
+          .block__header.block__header--light.page__header
+            .page__pagination = pagination
+            .page__info
+              span.block__header__title
+                = render PhilomenaWeb.PaginationView, "_pagination_info.html", page: @filters, conn: @conn
+
+      - assigns[:error] ->
+        p
+          ' Oops, there was an error evaluating your query:
+        pre = assigns[:error]
+
+      - true ->
+        p
+          ' No filters found!
+
+h3 Allowed fields
+table.table
+  thead
+    tr
+      th Field Selector
+      th Type
+      th Description
+      th Example
+  tbody
+    tr
+      td
+        code creator
+      td Literal
+      td Matches the creator of this filter.
+      td
+        code = link "creator:AppleDash", to: Routes.filter_path(@conn, :index, fq: "creator:AppleDash")
+    tr
+      td
+        code name
+      td Literal
+      td Matches the name of this filter. This is the default field.
+      td
+        code = link "name:default", to: Routes.filter_path(@conn, :index, fq: "name:default")
+    tr
+      td
+        code description
+      td Full Text
+      td Matches the description of this filter.
+      td
+        code = link "description:show's rating", to: Routes.filter_path(@conn, :index, fq: "description:the show's rating")
+    tr
+      td
+        code created_at
+      td Date/Time Range
+      td Matches the creation time of this filter.
+      td
+        code = link "created_at:2015", to: Routes.filter_path(@conn, :index, fq: "created_at:2015")
+    tr
+      td
+        code id
+      td Numeric Range
+      td Matches the numeric surrogate key for this filter.
+      td
+        code = link "id:1", to: Routes.filter_path(@conn, :index, fq: "id:1")
+    tr
+      td
+        code spoilered_count
+      td Numeric Range
+      td Matches the number of spoilered tags in this filter.
+      td
+        code = link "spoilered_count:1", to: Routes.filter_path(@conn, :index, fq: "spoilered_count:1")
+    tr
+      td
+        code hidden_count
+      td Numeric Range
+      td Matches the number of hidden tags in this filter.
+      td
+        code = link "hidden_count:1", to: Routes.filter_path(@conn, :index, fq: "hidden_count:1")
+    tr
+      td
+        code my
+      td Meta
+      td
+        code> my:filters
+        ' matches filters you have published if you are signed in.
+      td
+        code = link "my:filters", to: Routes.filter_path(@conn, :index, fq: "my:filters")
+    tr
+      td
+        code system
+      td Boolean
+      td Matches system filters
+      td
+        code = link "system:true", to: Routes.filter_path(@conn, :index, fq: "system:true")
+    tr
+      td
+        code public
+      td Boolean
+      td
+        ' Matches public filters. Note that
+        code> public:false
+        ' matches only your own private filters.
+      td
+        code = link "public:false", to: Routes.filter_path(@conn, :index, fq: "public:false")
+    tr
+      td
+        code user_id
+      td Literal
+      td Matches filters with the specified user_id.
+      td
+        code = link "user_id:307505", to: Routes.filter_path(@conn, :index, fq: "user_id:307505")
+
+= if @conn.params["fq"] do
+  p = link("Back to filters", to: Routes.filter_path(@conn, :index))

priv/repo/seeds.exs

@@ -19,6 +19,7 @@ alias Philomena.{
   Posts.Post,
   Images.Image,
   Reports.Report,
+  Filters.Filter,
   Roles.Role,
   Tags.Tag,
   Users.User,
@@ -28,11 +29,12 @@ alias Philomena.{
 alias Philomena.Elasticsearch
 alias Philomena.Users
 alias Philomena.Tags
+alias Philomena.Filters
 import Ecto.Query
 
 IO.puts("---- Creating Elasticsearch indices")
 
-for model <- [Image, Comment, Gallery, Tag, Post, Report] do
+for model <- [Image, Comment, Gallery, Tag, Post, Report, Filter] do
   Elasticsearch.delete_index!(model)
   Elasticsearch.create_index!(model)
 end
@@ -64,6 +66,13 @@ for filter_def <- resources["system_filters"] do
     hidden_tag_list: hidden_tag_list
   })
   |> Repo.insert(on_conflict: :nothing)
+  |> case do
+    {:ok, filter} ->
+      Filters.reindex_filter(filter)
+
+    {:error, changeset} ->
+      IO.inspect(changeset.errors)
+  end
 end
 
 IO.puts("---- Generating forums")