PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2024-11-27 05:37:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Public filter search (#88)

Browse source
This commit is contained in:
VcSaJen 2021-01-19 03:00:35 +09:00 committed by GitHub
parent 396ecafa6c
commit 24b22f78be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 525 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
index/all.mk
View file

@ -1,4 +1,4 @@
all: comments galleries images posts reports tags
all: comments galleries images posts reports tags filters
comments:
$(MAKE) -f comments.mk
@ -18,5 +18,8 @@ reports:
tags:
$(MAKE) -f tags.mk
filters:
$(MAKE) -f filters.mk
clean:
rm -f ./*.jsonl

50
index/filters.mk Normal file
View file

@ -0,0 +1,50 @@
DATABASE ?= philomena
ELASTICSEARCH_URL ?= http://localhost:9200/
ELASTICDUMP ?= elasticdump
# uncomment if getting "redirection unexpected" error on dump_jsonl
#SHELL=/bin/bash
.ONESHELL:
all: import_es
import_es: dump_jsonl
$(ELASTICDUMP) --input=filters.jsonl --output=$(ELASTICSEARCH_URL) --output-index=filters --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id"
dump_jsonl: metadata creators
psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_filters.jsonb_object_agg(object) from temp_filters.filter_search_json group by filter_id) to stdout;' > filters.jsonl
psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'drop schema temp_filters cascade;'
sed -i filters.jsonl -e 's/\\\\/\\/g'
metadata: filter_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
insert into temp_filters.filter_search_json (filter_id, object) select f.id, jsonb_build_object(
'id', f.id,
'created_at', f.created_at,
'user_id', f.user_id,
'public', f.public,
'system', f.system,
'name', lower(f.name),
'description', f.description,
'spoilered_count', array_length(f.spoilered_tag_ids, 1),
'hidden_count', array_length(f.hidden_tag_ids, 1),
'spoilered_tag_ids', f.spoilered_tag_ids,
'hidden_tag_ids', f.hidden_tag_ids,
'spoilered_complex_str', lower(f.spoilered_complex_str),
'hidden_complex_str', lower(f.hidden_complex_str),
'user_count', f.user_count
) from filters f;
SQL
creators: filter_search_json
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
insert into temp_filters.filter_search_json (filter_id, object) select f.id, jsonb_build_object('creator', lower(u.name)) from filters f left join users u on f.user_id=u.id;
SQL
filter_search_json:
psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
drop schema if exists temp_filters cascade;
create schema temp_filters;
create unlogged table temp_filters.filter_search_json (filter_id bigint not null, object jsonb not null);
create or replace aggregate temp_filters.jsonb_object_agg(jsonb) (sfunc = 'jsonb_concat', stype = jsonb, initcond='{}');
SQL

8
lib/mix/tasks/reindex_all.ex
View file

@ -9,10 +9,11 @@ defmodule Mix.Tasks.ReindexAll do
Posts.Post,
Images.Image,
Reports.Report,
Tags.Tag
Tags.Tag,
Filters.Filter
}
alias Philomena.{Comments, Galleries, Posts, Images, Tags}
alias Philomena.{Comments, Galleries, Posts, Images, Tags, Filters}
alias Philomena.Polymorphic
alias Philomena.Repo
import Ecto.Query
@ -30,7 +31,8 @@ defmodule Mix.Tasks.ReindexAll do
{Comments, Comment},
{Galleries, Gallery},
{Tags, Tag},
{Posts, Post}
{Posts, Post},
{Filters, Filter}
] do
Elasticsearch.delete_index!(schema)
Elasticsearch.create_index!(schema)

3
lib/philomena/elasticsearch.ex
View file

@ -11,6 +11,7 @@ defmodule Philomena.Elasticsearch do
alias Philomena.Posts.Post
alias Philomena.Reports.Report
alias Philomena.Tags.Tag
alias Philomena.Filters.Filter
alias Philomena.Comments.ElasticsearchIndex, as: CommentIndex
alias Philomena.Galleries.ElasticsearchIndex, as: GalleryIndex
@ -18,6 +19,7 @@ defmodule Philomena.Elasticsearch do
alias Philomena.Posts.ElasticsearchIndex, as: PostIndex
alias Philomena.Reports.ElasticsearchIndex, as: ReportIndex
alias Philomena.Tags.ElasticsearchIndex, as: TagIndex
alias Philomena.Filters.ElasticsearchIndex, as: FilterIndex
defp index_for(Comment), do: CommentIndex
defp index_for(Gallery), do: GalleryIndex
@ -25,6 +27,7 @@ defmodule Philomena.Elasticsearch do
defp index_for(Post), do: PostIndex
defp index_for(Report), do: ReportIndex
defp index_for(Tag), do: TagIndex
defp index_for(Filter), do: FilterIndex
defp elastic_url do
Application.get_env(:philomena, :elasticsearch_url)

58
lib/philomena/filters.ex
View file

@ -7,6 +7,9 @@ defmodule Philomena.Filters do
alias Philomena.Repo
alias Philomena.Filters.Filter
alias Philomena.Elasticsearch
alias Philomena.Filters.ElasticsearchIndex, as: FilterIndex
alias Philomena.IndexWorker
@doc """
Returns the list of filters.
@ -68,6 +71,7 @@ defmodule Philomena.Filters do
%Filter{user_id: user.id}
|> Filter.creation_changeset(attrs)
|> Repo.insert()
|> reindex_after_update()
end
@doc """
@ -86,12 +90,14 @@ defmodule Philomena.Filters do
filter
|> Filter.update_changeset(attrs)
|> Repo.update()
|> reindex_after_update()
end
def make_filter_public(%Filter{} = filter) do
filter
|> Filter.public_changeset()
|> Repo.update()
|> reindex_after_update()
end
@doc """
@ -110,6 +116,15 @@ defmodule Philomena.Filters do
filter
|> Filter.deletion_changeset()
|> Repo.delete()
|> case do
{:ok, filter} ->
unindex_filter(filter)
{:ok, filter}
error ->
error
end
end
@doc """
@ -162,6 +177,7 @@ defmodule Philomena.Filters do
filter
|> Filter.hidden_tags_changeset(hidden_tag_ids)
|> Repo.update()
|> reindex_after_update()
end
def unhide_tag(filter, tag) do
@ -170,6 +186,7 @@ defmodule Philomena.Filters do
filter
|> Filter.hidden_tags_changeset(hidden_tag_ids)
|> Repo.update()
|> reindex_after_update()
end
def spoiler_tag(filter, tag) do
@ -178,6 +195,7 @@ defmodule Philomena.Filters do
filter
|> Filter.spoilered_tags_changeset(spoilered_tag_ids)
|> Repo.update()
|> reindex_after_update()
end
def unspoiler_tag(filter, tag) do
@ -186,5 +204,45 @@ defmodule Philomena.Filters do
filter
|> Filter.spoilered_tags_changeset(spoilered_tag_ids)
|> Repo.update()
|> reindex_after_update()
end
defp reindex_after_update({:ok, filter}) do
reindex_filter(filter)
{:ok, filter}
end
defp reindex_after_update(error) do
error
end
def user_name_reindex(old_name, new_name) do
data = FilterIndex.user_name_update_by_query(old_name, new_name)
Elasticsearch.update_by_query(Filter, data.query, data.set_replacements, data.replacements)
end
def reindex_filter(%Filter{} = filter) do
Exq.enqueue(Exq, "indexing", IndexWorker, ["Filters", "id", [filter.id]])
filter
end
def unindex_filter(%Filter{} = filter) do
Elasticsearch.delete_document(filter.id, Filter)
filter
end
def indexing_preloads do
[:user]
end
def perform_reindex(column, condition) do
Filter
|> preload(^indexing_preloads())
|> where([f], field(f, ^column) in ^condition)
|> Elasticsearch.reindex(Filter)
end
end

73
lib/philomena/filters/elasticsearch_index.ex Normal file
View file

@ -0,0 +1,73 @@
defmodule Philomena.Filters.ElasticsearchIndex do
@behaviour Philomena.ElasticsearchIndex
@impl true
def index_name do
"filters"
end
@impl true
def mapping do
%{
settings: %{
index: %{
number_of_shards: 5,
max_result_window: 10_000_000
}
},
mappings: %{
dynamic: false,
properties: %{
id: %{type: "integer"},
created_at: %{type: "date"},
user_id: %{type: "keyword"},
creator: %{type: "keyword"},
public: %{type: "boolean"},
system: %{type: "boolean"},
name: %{type: "keyword"},
description: %{type: "text", analyzer: "snowball"},
spoilered_count: %{type: "integer"},
hidden_count: %{type: "integer"},
spoilered_tag_ids: %{type: "keyword"},
hidden_tag_ids: %{type: "keyword"},
spoilered_tags: %{type: "keyword"},
hidden_tags: %{type: "keyword"},
spoilered_complex_str: %{type: "keyword"},
hidden_complex_str: %{type: "keyword"},
user_count: %{type: "integer"}
}
}
}
end
@impl true
def as_json(filter) do
%{
id: filter.id,
created_at: filter.created_at,
user_id: filter.user_id,
creator: if(!!filter.user, do: String.downcase(filter.user.name)),
public: filter.public || filter.system,
system: filter.system,
name: filter.name |> String.downcase(),
description: filter.description,
spoilered_count: length(filter.spoilered_tag_ids),
hidden_count: length(filter.hidden_tag_ids),
spoilered_tag_ids: filter.spoilered_tag_ids,
hidden_tag_ids: filter.hidden_tag_ids,
spoilered_complex_str:
if(!!filter.spoilered_complex_str, do: String.downcase(filter.spoilered_complex_str)),
hidden_complex_str:
if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)),
user_count: filter.user_count
}
end
def user_name_update_by_query(old_name, new_name) do
%{
query: %{term: %{creator: old_name}},
replacements: [%{path: ["creator"], old: old_name, new: new_name}],
set_replacements: []
}
end
end

47
lib/philomena/filters/query.ex Normal file
View file

@ -0,0 +1,47 @@
defmodule Philomena.Filters.Query do
alias Philomena.Search.Parser
defp user_my_transform(%{user: %{id: id}}, "filters"),
do: {:ok, %{term: %{user_id: id}}}
defp user_my_transform(_ctx, _value),
do: {:error, "Unknown `my' value."}
defp anonymous_fields do
[
int_fields: ~W(id spoilered_count hidden_count),
date_fields: ~W(created_at),
ngram_fields: ~W(description),
literal_fields: ~W(name creator user_id),
bool_fields: ~W(public system),
default_field: {"name", :term}
]
end
defp user_fields do
fields = anonymous_fields()
Keyword.merge(fields,
custom_fields: ~W(my),
transforms: %{"my" => &user_my_transform/2}
)
end
defp parse(fields, context, query_string) do
fields
|> Parser.parser()
|> Parser.parse(query_string, context)
end
def compile(user, query_string) do
query_string = query_string || ""
case user do
nil ->
parse(anonymous_fields(), %{user: nil}, query_string)
user ->
parse(user_fields(), %{user: user}, query_string)
end
end
end

7
lib/philomena/tags.ex
View file

@ -17,6 +17,7 @@ defmodule Philomena.Tags do
alias Philomena.Images
alias Philomena.Images.Image
alias Philomena.Users.User
alias Philomena.Filters
alias Philomena.Filters.Filter
alias Philomena.Images.Tagging
alias Philomena.ArtistLinks.ArtistLink
@ -301,6 +302,12 @@ defmodule Philomena.Tags do
|> where([_i, t], t.id == ^tag.id)
|> preload(^Images.indexing_preloads())
|> Elasticsearch.reindex(Image)
Filter
|> where([f], fragment("? @> ARRAY[?]::integer[]", f.hidden_tag_ids, ^tag.id))
|> or_where([f], fragment("? @> ARRAY[?]::integer[]", f.spoilered_tag_ids, ^tag.id))
|> preload(^Filters.indexing_preloads())
|> Elasticsearch.reindex(Filter)
end
def unalias_tag(%Tag{} = tag) do

2
lib/philomena/users.ex
View file

@ -17,6 +17,7 @@ defmodule Philomena.Users do
alias Philomena.Posts
alias Philomena.Galleries
alias Philomena.Reports
alias Philomena.Filters
alias Philomena.UserRenameWorker
## Database getters
@ -620,6 +621,7 @@ defmodule Philomena.Users do
Posts.user_name_reindex(old_name, new_name)
Galleries.user_name_reindex(old_name, new_name)
Reports.user_name_reindex(old_name, new_name)
Filters.user_name_reindex(old_name, new_name)
end
def reactivate_user(%User{} = user) do

3
lib/philomena/workers/index_worker.ex
View file

@ -5,7 +5,8 @@ defmodule Philomena.IndexWorker do
"Images" => Philomena.Images,
"Posts" => Philomena.Posts,
"Reports" => Philomena.Reports,
"Tags" => Philomena.Tags
"Tags" => Philomena.Tags,
"Filters" => Philomena.Filters
}
# Perform the queued index. Context function looks like the following:

61
lib/philomena_web/controllers/api/json/search/filter_controller.ex Normal file
View file

@ -0,0 +1,61 @@
defmodule PhilomenaWeb.Api.Json.Search.FilterController do
use PhilomenaWeb, :controller
alias Philomena.Elasticsearch
alias Philomena.Filters.Filter
alias Philomena.Filters.Query
import Ecto.Query
def index(conn, params) do
user = conn.assigns.current_user
case Query.compile(user, params["q"] || "") do
{:ok, query} ->
filters =
Filter
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
must: [
query,
%{
bool: %{
should:
[%{term: %{public: true}}, %{term: %{system: true}}] ++
user_should(user)
}
}
]
}
},
sort: [
%{name: :asc},
%{id: :desc}
]
},
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Filter, [:user]))
conn
|> put_view(PhilomenaWeb.Api.Json.FilterView)
|> render("index.json", filters: filters, total: filters.total_entries)
{:error, msg} ->
conn
|> put_status(:bad_request)
|> json(%{error: msg})
end
end
defp user_should(user) do
case user do
nil ->
[]
user ->
[%{term: %{user_id: user.id}}]
end
end
end

56
lib/philomena_web/controllers/filter_controller.ex
View file

@ -1,7 +1,8 @@
defmodule PhilomenaWeb.FilterController do
use PhilomenaWeb, :controller
alias Philomena.{Filters, Filters.Filter, Tags.Tag}
alias Philomena.{Filters, Filters.Filter, Filters.Query, Tags.Tag}
alias Philomena.Elasticsearch
alias Philomena.Schema.TagList
alias Philomena.Repo
import Ecto.Query
@ -9,6 +10,14 @@ defmodule PhilomenaWeb.FilterController do
plug :load_and_authorize_resource, model: Filter, except: [:index], preload: :user
plug PhilomenaWeb.RequireUserPlug when action not in [:index, :show]
def index(conn, %{"fq" => fq}) do
user = conn.assigns.current_user
user
|> Query.compile(fq)
|> render_index(conn, user)
end
def index(conn, _params) do
user = conn.assigns.current_user
@ -35,6 +44,51 @@ defmodule PhilomenaWeb.FilterController do
)
end
defp render_index({:ok, query}, conn, user) do
filters =
Filter
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
must: [query | filters(user)]
}
},
sort: [
%{name: :asc},
%{id: :desc}
]
},
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Filter, [:user]))
render(conn, "index.html", title: "Filters", filters: filters)
end
defp render_index({:error, msg}, conn, _user) do
render(conn, "index.html", title: "Filters", error: msg, filters: [])
end
defp filters(user),
do: [%{bool: %{should: shoulds(user)}}]
defp shoulds(user) do
case user do
nil ->
anonymous_should()
user ->
user_should(user)
end
end
defp user_should(user),
do: anonymous_should() ++ [%{term: %{user_id: user.id}}]
defp anonymous_should(),
do: [%{term: %{public: true}}, %{term: %{system: true}}]
def show(conn, _params) do
filter = conn.assigns.filter

1
lib/philomena_web/router.ex
View file

@ -122,6 +122,7 @@ defmodule PhilomenaWeb.Router do
resources "/posts", PostController, only: [:index]
resources "/comments", CommentController, only: [:index]
resources "/galleries", GalleryController, only: [:index]
resources "/filters", FilterController, only: [:index]
end
# Convenience alias

135
lib/philomena_web/templates/filter/index.html.slime
View file

@ -24,6 +24,7 @@
p
' By default all the filters you create are private and only visible by you. You can have as many as you like and switch between them instantly with no limits. You can also create a public filter, which can be seen and used by any user on the site, allowing you to share useful filters with others.
= if !@conn.params["fq"] do
h2 My Filters
= if @current_user do
p
@ -37,3 +38,137 @@
h2 Global Filters
= for filter <- @system_filters do
= render PhilomenaWeb.FilterView, "_filter.html", conn: @conn, filter: filter
h2 Search Filters
p
' Some users maintain custom filters which are publicly shared; you can search these filters with the box below.
= form_for :filters, Routes.filter_path(@conn, :index), [method: "get", class: "hform", enforce_utf8: false], fn f ->
.field
= text_input f, :fq, name: :fq, value: @conn.params["fq"], class: "input hform__text", placeholder: "Search filters", autocapitalize: "none"
= submit "Search", class: "hform__button button"
.fieldlabel
' For more information, see the
a href="/pages/search_syntax" search syntax documentation
' . Search results are sorted by creation date.
= if @conn.params["fq"] do
h2 Search Results
= cond do
- Enum.any?(@filters) ->
- route = fn p -> Routes.filter_path(@conn, :index, p) end
- pagination = render PhilomenaWeb.PaginationView, "_pagination.html", page: @filters, route: route, params: [fq: @conn.params["fq"]], conn: @conn
= for filter <- @filters do
= render PhilomenaWeb.FilterView, "_filter.html", conn: @conn, filter: filter
.block
.block__header.block__header--light.page__header
.page__pagination = pagination
.page__info
span.block__header__title
= render PhilomenaWeb.PaginationView, "_pagination_info.html", page: @filters, conn: @conn
- assigns[:error] ->
p
' Oops, there was an error evaluating your query:
pre = assigns[:error]
- true ->
p
' No filters found!
h3 Allowed fields
table.table
thead
tr
th Field Selector
th Type
th Description
th Example
tbody
tr
td
code creator
td Literal
td Matches the creator of this filter.
td
code = link "creator:AppleDash", to: Routes.filter_path(@conn, :index, fq: "creator:AppleDash")
tr
td
code name
td Literal
td Matches the name of this filter. This is the default field.
td
code = link "name:default", to: Routes.filter_path(@conn, :index, fq: "name:default")
tr
td
code description
td Full Text
td Matches the description of this filter.
td
code = link "description:show's rating", to: Routes.filter_path(@conn, :index, fq: "description:the show's rating")
tr
td
code created_at
td Date/Time Range
td Matches the creation time of this filter.
td
code = link "created_at:2015", to: Routes.filter_path(@conn, :index, fq: "created_at:2015")
tr
td
code id
td Numeric Range
td Matches the numeric surrogate key for this filter.
td
code = link "id:1", to: Routes.filter_path(@conn, :index, fq: "id:1")
tr
td
code spoilered_count
td Numeric Range
td Matches the number of spoilered tags in this filter.
td
code = link "spoilered_count:1", to: Routes.filter_path(@conn, :index, fq: "spoilered_count:1")
tr
td
code hidden_count
td Numeric Range
td Matches the number of hidden tags in this filter.
td
code = link "hidden_count:1", to: Routes.filter_path(@conn, :index, fq: "hidden_count:1")
tr
td
code my
td Meta
td
code> my:filters
' matches filters you have published if you are signed in.
td
code = link "my:filters", to: Routes.filter_path(@conn, :index, fq: "my:filters")
tr
td
code system
td Boolean
td Matches system filters
td
code = link "system:true", to: Routes.filter_path(@conn, :index, fq: "system:true")
tr
td
code public
td Boolean
td
' Matches public filters. Note that
code> public:false
' matches only your own private filters.
td
code = link "public:false", to: Routes.filter_path(@conn, :index, fq: "public:false")
tr
td
code user_id
td Literal
td Matches filters with the specified user_id.
td
code = link "user_id:307505", to: Routes.filter_path(@conn, :index, fq: "user_id:307505")
= if @conn.params["fq"] do
p = link("Back to filters", to: Routes.filter_path(@conn, :index))

11
priv/repo/seeds.exs
View file

@ -19,6 +19,7 @@ alias Philomena.{
Posts.Post,
Images.Image,
Reports.Report,
Filters.Filter,
Roles.Role,
Tags.Tag,
Users.User,
@ -28,11 +29,12 @@ alias Philomena.{
alias Philomena.Elasticsearch
alias Philomena.Users
alias Philomena.Tags
alias Philomena.Filters
import Ecto.Query
IO.puts("---- Creating Elasticsearch indices")
for model <- [Image, Comment, Gallery, Tag, Post, Report] do
for model <- [Image, Comment, Gallery, Tag, Post, Report, Filter] do
Elasticsearch.delete_index!(model)
Elasticsearch.create_index!(model)
end
@ -64,6 +66,13 @@ for filter_def <- resources["system_filters"] do
hidden_tag_list: hidden_tag_list
})
|> Repo.insert(on_conflict: :nothing)
|> case do
{:ok, filter} ->
Filters.reindex_filter(filter)
{:error, changeset} ->
IO.inspect(changeset.errors)
end
end
IO.puts("---- Generating forums")