separate search definition building from search execution

This commit is contained in:
byte[] 2020-08-23 15:47:42 -04:00
parent 42d03ef548
commit 3fce31b658
23 changed files with 151 additions and 100 deletions

View file

@ -183,7 +183,7 @@ defmodule Philomena.Elasticsearch do
results
end
def search_results(module, elastic_query, pagination_params \\ %{}) do
def search_definition(module, elastic_query, pagination_params \\ %{}) do
page_number = pagination_params[:page_number] || 1
page_size = pagination_params[:page_size] || 25
@ -195,25 +195,34 @@ defmodule Philomena.Elasticsearch do
track_total_hits: true
})
results = search(module, elastic_query)
%{
module: module,
body: elastic_query,
page_number: page_number,
page_size: page_size
}
end
def search_results(definition) do
results = search(definition.module, definition.body)
time = results["took"]
count = results["hits"]["total"]["value"]
entries = Enum.map(results["hits"]["hits"], &{String.to_integer(&1["_id"]), &1})
Logger.debug("[Elasticsearch] Query took #{time}ms")
Logger.debug("[Elasticsearch] #{Jason.encode!(elastic_query)}")
Logger.debug("[Elasticsearch] #{Jason.encode!(definition.body)}")
%Scrivener.Page{
entries: entries,
page_number: page_number,
page_size: page_size,
page_number: definition.page_number,
page_size: definition.page_size,
total_entries: count,
total_pages: div(count + page_size - 1, page_size)
total_pages: div(count + definition.page_size - 1, definition.page_size)
}
end
def search_records_with_hits(module, elastic_query, pagination_params, ecto_query) do
page = search_results(module, elastic_query, pagination_params)
defp load_records_from_results(results, ecto_queries) do
Enum.map(Enum.zip(results, ecto_queries), fn {page, ecto_query} ->
{ids, hits} = Enum.unzip(page.entries)
records =
@ -223,10 +232,17 @@ defmodule Philomena.Elasticsearch do
|> Enum.sort_by(&Enum.find_index(ids, fn el -> el == &1.id end))
%{page | entries: Enum.zip(records, hits)}
end)
end
def search_records(module, elastic_query, pagination_params, ecto_query) do
page = search_records_with_hits(module, elastic_query, pagination_params, ecto_query)
def search_records_with_hits(definition, ecto_query) do
[page] = load_records_from_results([search_results(definition)], [ecto_query])
page
end
def search_records(definition, ecto_query) do
page = search_records_with_hits(definition, ecto_query)
{records, _hits} = Enum.unzip(page.entries)
%{page | entries: records}

View file

@ -35,9 +35,12 @@ defmodule PhilomenaWeb.ActivityController do
pagination: %{page_number: :rand.uniform(6), page_size: 4}
)
images = Elasticsearch.search_records(images, preload(Image, :tags))
top_scoring = Elasticsearch.search_records(top_scoring, preload(Image, :tags))
comments =
Elasticsearch.search_records(
Comment,
Comment
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -52,9 +55,9 @@ defmodule PhilomenaWeb.ActivityController do
},
sort: %{posted_at: :desc}
},
%{page_number: 1, page_size: 6},
Comment |> preload([:user, image: [:tags]])
%{page_number: 1, page_size: 6}
)
|> Elasticsearch.search_records(preload(Comment, [:user, image: [:tags]]))
watched =
if !!user do
@ -65,6 +68,8 @@ defmodule PhilomenaWeb.ActivityController do
pagination: %{conn.assigns.image_pagination | page_number: 1}
)
watched_images = Elasticsearch.search_records(watched_images, preload(Image, :tags))
if Enum.any?(watched_images), do: watched_images
end

View file

@ -80,15 +80,15 @@ defmodule PhilomenaWeb.Admin.ReportController do
defp load_reports(conn, query) do
reports =
Elasticsearch.search_records(
Report,
Report
|> Elasticsearch.search_definition(
%{
query: query,
sort: sorts()
},
conn.assigns.pagination,
Report |> preload([:admin, user: :linked_tags])
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Report, [:admin, user: :linked_tags]))
entries = Polymorphic.load_polymorphic(reports, reportable: [reportable_id: :reportable_type])

View file

@ -13,8 +13,8 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do
case Query.compile(user, params["q"] || "") do
{:ok, query} ->
comments =
Elasticsearch.search_records(
Comment,
Comment
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -29,9 +29,9 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do
},
sort: %{posted_at: :desc}
},
conn.assigns.pagination,
preload(Comment, [:image, :user])
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Comment, [:image, :user]))
conn
|> put_view(PhilomenaWeb.Api.Json.CommentView)

View file

@ -10,15 +10,15 @@ defmodule PhilomenaWeb.Api.Json.Search.GalleryController do
case Query.compile(params["q"] || "") do
{:ok, query} ->
galleries =
Elasticsearch.search_records(
Gallery,
Gallery
|> Elasticsearch.search_definition(
%{
query: query,
sort: %{created_at: :desc}
},
conn.assigns.pagination,
preload(Gallery, [:creator])
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Gallery, [:creator]))
conn
|> put_view(PhilomenaWeb.Api.Json.GalleryView)

View file

@ -2,6 +2,7 @@ defmodule PhilomenaWeb.Api.Json.Search.ImageController do
use PhilomenaWeb, :controller
alias PhilomenaWeb.ImageLoader
alias Philomena.Elasticsearch
alias Philomena.Interactions
alias Philomena.Images.Image
import Ecto.Query
@ -10,8 +11,9 @@ defmodule PhilomenaWeb.Api.Json.Search.ImageController do
queryable = Image |> preload([:tags, :user, :intensity])
user = conn.assigns.current_user
case ImageLoader.search_string(conn, params["q"], queryable: queryable) do
case ImageLoader.search_string(conn, params["q"]) do
{:ok, {images, _tags}} ->
images = Elasticsearch.search_records(images, queryable)
interactions = Interactions.user_interactions(images, user)
conn

View file

@ -12,8 +12,8 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do
case Query.compile(user, params["q"] || "") do
{:ok, query} ->
posts =
Elasticsearch.search_records(
Post,
Post
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -26,9 +26,9 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do
},
sort: %{created_at: :desc}
},
conn.assigns.pagination,
preload(Post, [:user, :topic])
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Post, [:user, :topic]))
conn
|> put_view(PhilomenaWeb.Api.Json.Forum.Topic.PostView)

View file

@ -10,10 +10,12 @@ defmodule PhilomenaWeb.Api.Json.Search.TagController do
case Query.compile(params["q"] || "") do
{:ok, query} ->
tags =
Elasticsearch.search_records(
Tag,
Tag
|> Elasticsearch.search_definition(
%{query: query, sort: %{images: :desc}},
conn.assigns.pagination,
conn.assigns.pagination
)
|> Elasticsearch.search_records(
preload(Tag, [:aliased_tag, :aliases, :implied_tags, :implied_by_tags, :dnp_entries])
)

View file

@ -2,9 +2,14 @@ defmodule PhilomenaWeb.Api.Rss.WatchedController do
use PhilomenaWeb, :controller
alias PhilomenaWeb.ImageLoader
alias Philomena.Images.Image
alias Philomena.Elasticsearch
import Ecto.Query
def index(conn, _params) do
{:ok, {images, _tags}} = ImageLoader.search_string(conn, "my:watched")
images = Elasticsearch.search_records(images, preload(Image, :tags))
# NB: this is RSS, but using the RSS format causes Phoenix not to
# escape HTML

View file

@ -20,8 +20,8 @@ defmodule PhilomenaWeb.CommentController do
defp render_index({:ok, query}, conn, user) do
comments =
Elasticsearch.search_records(
Comment,
Comment
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -33,8 +33,10 @@ defmodule PhilomenaWeb.CommentController do
},
sort: %{posted_at: :desc}
},
conn.assigns.pagination,
Comment |> preload([:deleted_by, image: [:tags], user: [awards: :badge]])
conn.assigns.pagination
)
|> Elasticsearch.search_records(
preload(Comment, [:deleted_by, image: [:tags], user: [awards: :badge]])
)
rendered = TextileRenderer.render_collection(comments.entries, conn)

View file

@ -7,6 +7,7 @@ defmodule PhilomenaWeb.GalleryController do
alias Philomena.Interactions
alias Philomena.Galleries.Gallery
alias Philomena.Galleries
alias Philomena.Images.Image
import Ecto.Query
plug PhilomenaWeb.FilterBannedUsersPlug when action in [:new, :create, :edit, :update, :delete]
@ -19,8 +20,8 @@ defmodule PhilomenaWeb.GalleryController do
def index(conn, params) do
galleries =
Elasticsearch.search_records(
Gallery,
Gallery
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -29,9 +30,9 @@ defmodule PhilomenaWeb.GalleryController do
},
sort: parse_sort(params)
},
conn.assigns.pagination,
Gallery |> preload([:creator, thumbnail: :tags])
conn.assigns.pagination
)
|> Elasticsearch.search_records(preload(Gallery, [:creator, thumbnail: :tags]))
render(conn, "index.html",
title: "Galleries",
@ -54,7 +55,8 @@ defmodule PhilomenaWeb.GalleryController do
conn = %{conn | params: params}
{:ok, {images, _tags}} = ImageLoader.search_string(conn, query, include_hits: true)
{:ok, {images, _tags}} = ImageLoader.search_string(conn, query)
images = Elasticsearch.search_records_with_hits(images, preload(Image, :tags))
{gallery_prev, gallery_next} = prev_next_page_images(conn, query)
@ -163,7 +165,9 @@ defmodule PhilomenaWeb.GalleryController do
pagination_params = %{page_number: offset + 1, page_size: 1}
{:ok, {image, _tags}} =
ImageLoader.search_string(conn, query, pagination: pagination_params, include_hits: true)
ImageLoader.search_string(conn, query, pagination: pagination_params)
image = Elasticsearch.search_records_with_hits(image, preload(Image, :tags))
case Enum.to_list(image) do
[image] -> image

View file

@ -4,6 +4,7 @@ defmodule PhilomenaWeb.Image.NavigateController do
alias PhilomenaWeb.ImageLoader
alias PhilomenaWeb.ImageNavigator
alias PhilomenaWeb.ImageScope
alias Philomena.Elasticsearch
alias Philomena.Images.Image
alias Philomena.Images.Query
@ -35,7 +36,8 @@ defmodule PhilomenaWeb.Image.NavigateController do
# (although it probably should).
body = %{range: %{id: %{gt: conn.assigns.image.id}}}
{images, _tags} = ImageLoader.query(conn, body, queryable: Image, pagination: pagination)
{images, _tags} = ImageLoader.query(conn, body, pagination: pagination)
images = Elasticsearch.search_records(images, Image)
page_num = page_for_offset(pagination.page_size, images.total_entries)

View file

@ -33,8 +33,8 @@ defmodule PhilomenaWeb.Image.RandomController do
defp random_image_id(query, filter) do
%{query: query, sorts: sort} = ImageSorter.parse_sort(%{"sf" => "random"}, query)
Elasticsearch.search_records(
Image,
Image
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -47,9 +47,9 @@ defmodule PhilomenaWeb.Image.RandomController do
},
sort: sort
},
%{page_size: 1},
Image
%{page_size: 1}
)
|> Elasticsearch.search_records(Image)
|> Enum.to_list()
|> unwrap()
end

View file

@ -4,6 +4,8 @@ defmodule PhilomenaWeb.Image.RelatedController do
alias PhilomenaWeb.ImageLoader
alias Philomena.Interactions
alias Philomena.Images.Image
alias Philomena.Elasticsearch
import Ecto.Query
plug PhilomenaWeb.CanaryMapPlug, index: :show
@ -58,6 +60,8 @@ defmodule PhilomenaWeb.Image.RelatedController do
pagination: %{conn.assigns.image_pagination | page_number: 1}
)
images = Elasticsearch.search_records(images, preload(Image, :tags))
interactions = Interactions.user_interactions(images, user)
render(conn, "index.html",

View file

@ -13,6 +13,7 @@ defmodule PhilomenaWeb.ImageController do
Galleries.Gallery
}
alias Philomena.Elasticsearch
alias Philomena.Interactions
alias Philomena.Comments
alias Philomena.Repo
@ -36,6 +37,8 @@ defmodule PhilomenaWeb.ImageController do
def index(conn, _params) do
{:ok, {images, _tags}} = ImageLoader.search_string(conn, "created_at.lte:3 minutes ago")
images = Elasticsearch.search_records(images, preload(Image, :tags))
interactions = Interactions.user_interactions(images, conn.assigns.current_user)
render(conn, "index.html",

View file

@ -20,8 +20,8 @@ defmodule PhilomenaWeb.PostController do
defp render_index({:ok, query}, conn, user) do
posts =
Elasticsearch.search_records(
Post,
Post
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -30,8 +30,10 @@ defmodule PhilomenaWeb.PostController do
},
sort: %{created_at: :desc}
},
conn.assigns.pagination,
Post |> preload([:deleted_by, topic: :forum, user: [awards: :badge]])
conn.assigns.pagination
)
|> Elasticsearch.search_records(
preload(Post, [:deleted_by, topic: :forum, user: [awards: :badge]])
)
rendered = TextileRenderer.render_collection(posts.entries, conn)

View file

@ -16,6 +16,7 @@ defmodule PhilomenaWeb.ProfileController do
alias Philomena.UserFingerprints.UserFingerprint
alias Philomena.ModNotes.ModNote
alias Philomena.Polymorphic
alias Philomena.Images.Image
alias Philomena.Repo
import Ecto.Query
@ -52,6 +53,9 @@ defmodule PhilomenaWeb.ProfileController do
pagination: %{page_number: 1, page_size: 4}
)
recent_uploads = Elasticsearch.search_records(recent_uploads, preload(Image, :tags))
recent_faves = Elasticsearch.search_records(recent_faves, preload(Image, :tags))
tags = tags(conn.assigns.user.public_links)
all_tag_ids =
@ -74,8 +78,8 @@ defmodule PhilomenaWeb.ProfileController do
recent_artwork = recent_artwork(conn, tags)
recent_comments =
Elasticsearch.search_records(
Comment,
Comment
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -91,9 +95,9 @@ defmodule PhilomenaWeb.ProfileController do
},
sort: %{posted_at: :desc}
},
%{page_size: 3},
Comment |> preload(user: [awards: :badge], image: :tags)
%{page_size: 3}
)
|> Elasticsearch.search_records(preload(Comment, user: [awards: :badge], image: :tags))
|> Enum.filter(&Canada.Can.can?(current_user, :show, &1.image))
recent_comments =
@ -102,8 +106,8 @@ defmodule PhilomenaWeb.ProfileController do
|> Enum.zip(recent_comments)
recent_posts =
Elasticsearch.search_records(
Post,
Post
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -117,9 +121,9 @@ defmodule PhilomenaWeb.ProfileController do
},
sort: %{created_at: :desc}
},
%{page_size: 6},
Post |> preload(user: [awards: :badge], topic: :forum)
%{page_size: 6}
)
|> Elasticsearch.search_records(preload(Post, user: [awards: :badge], topic: :forum))
|> Enum.filter(&Canada.Can.can?(current_user, :show, &1.topic))
about_me = TextileRenderer.render_one(%{body: user.description || ""}, conn)
@ -220,7 +224,7 @@ defmodule PhilomenaWeb.ProfileController do
pagination: %{page_number: 1, page_size: 4}
)
images
Elasticsearch.search_records(images, preload(Image, :tags))
end
defp set_admin_metadata(conn, _opts) do

View file

@ -2,13 +2,17 @@ defmodule PhilomenaWeb.SearchController do
use PhilomenaWeb, :controller
alias PhilomenaWeb.ImageLoader
alias Philomena.Images.Image
alias Philomena.Elasticsearch
alias Philomena.Interactions
import Ecto.Query
def index(conn, params) do
user = conn.assigns.current_user
case ImageLoader.search_string(conn, params["q"], include_hits: custom_ordering?(conn)) do
case ImageLoader.search_string(conn, params["q"]) do
{:ok, {images, tags}} ->
images = search_function(custom_ordering?(conn)).(images, preload(Image, :tags))
interactions = Interactions.user_interactions(images, user)
conn
@ -31,6 +35,9 @@ defmodule PhilomenaWeb.SearchController do
end
end
defp search_function(true), do: &Elasticsearch.search_records_with_hits/2
defp search_function(_custom), do: &Elasticsearch.search_records/2
defp custom_ordering?(%{params: %{"sf" => sf}}) when sf != "id", do: true
defp custom_ordering?(_conn), do: false
end

View file

@ -12,8 +12,8 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do
[]
term ->
Elasticsearch.search_records(
Tag,
Tag
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -25,9 +25,9 @@ defmodule PhilomenaWeb.Tag.AutocompleteController do
},
sort: %{images: :desc}
},
%{page_size: 5},
Tag |> preload(:aliased_tag)
%{page_size: 5}
)
|> Elasticsearch.search_records(preload(Tag, :aliased_tag))
|> Enum.map(&(&1.aliased_tag || &1))
|> Enum.uniq_by(& &1.id)
|> Enum.sort_by(&(-&1.images_count))

View file

@ -4,8 +4,10 @@ defmodule PhilomenaWeb.TagController do
alias PhilomenaWeb.ImageLoader
alias Philomena.Elasticsearch
alias Philomena.{Tags, Tags.Tag}
alias Philomena.Images.Image
alias PhilomenaWeb.TextileRenderer
alias Philomena.Interactions
import Ecto.Query
plug PhilomenaWeb.RecodeParameterPlug, [name: "id"] when action in [:show]
plug PhilomenaWeb.CanaryMapPlug, update: :edit
@ -32,16 +34,16 @@ defmodule PhilomenaWeb.TagController do
with {:ok, query} <- Tags.Query.compile(query_string) do
tags =
Elasticsearch.search_records(
Tag,
Tag
|> Elasticsearch.search_definition(
%{
query: query,
size: 250,
sort: [%{images: :desc}, %{name: :asc}]
},
%{conn.assigns.pagination | page_size: 250},
Tag
%{conn.assigns.pagination | page_size: 250}
)
|> Elasticsearch.search_records(Tag)
render(conn, "index.html", title: "Tags", tags: tags)
else
@ -56,6 +58,8 @@ defmodule PhilomenaWeb.TagController do
{images, _tags} = ImageLoader.query(conn, %{term: %{"namespaced_tags.name" => tag.name}})
images = Elasticsearch.search_records(images, preload(Image, :tags))
interactions = Interactions.user_interactions(images, user)
body = TextileRenderer.render_one(%{body: tag.description || ""}, conn)

View file

@ -20,7 +20,6 @@ defmodule PhilomenaWeb.ImageLoader do
def query(conn, body, options \\ []) do
pagination = Keyword.get(options, :pagination, conn.assigns.image_pagination)
queryable = Keyword.get(options, :queryable, Image |> preload(:tags))
sorts = Keyword.get(options, :sorts, &ImageSorter.parse_sort(conn.params, &1))
tags =
@ -35,8 +34,8 @@ defmodule PhilomenaWeb.ImageLoader do
%{query: query, sorts: sort} = sorts.(body)
records =
search_function(options).(
definition =
Elasticsearch.search_definition(
Image,
%{
query: %{
@ -47,11 +46,10 @@ defmodule PhilomenaWeb.ImageLoader do
},
sort: sort
},
pagination,
queryable
pagination
)
{records, tags}
{definition, tags}
end
defp create_filters(conn, user, filter) do
@ -95,17 +93,6 @@ defmodule PhilomenaWeb.ImageLoader do
defp maybe_custom_hide(filters, _user, _param),
do: filters
# Allow callers to choose if they want inner hit objects returned;
# primarily useful for allowing client navigation through images
@spec search_function(Keyword.t()) :: function()
defp search_function(options) do
case Keyword.get(options, :include_hits) do
true -> &Elasticsearch.search_records_with_hits/4
_false -> &Elasticsearch.search_records/4
end
end
# TODO: the search parser should try to optimize queries
defp search_tag_name(%{term: %{"namespaced_tags.name" => tag_name}}), do: [tag_name]
defp search_tag_name(_other_query), do: []

View file

@ -51,7 +51,9 @@ defmodule PhilomenaWeb.ImageNavigator do
end
defp maybe_search_after(module, body, options, queryable, true) do
Elasticsearch.search_records_with_hits(module, body, options, queryable)
module
|> Elasticsearch.search_definition(body, options)
|> Elasticsearch.search_records_with_hits(queryable)
end
defp maybe_search_after(_module, _body, _options, _queryable, _false) do

View file

@ -132,8 +132,8 @@ defmodule PhilomenaWeb.TagView do
end
defp implied_by_multitag(tag_names, ignore_tag_names) do
Elasticsearch.search_records(
Tag,
Tag
|> Elasticsearch.search_definition(
%{
query: %{
bool: %{
@ -143,9 +143,9 @@ defmodule PhilomenaWeb.TagView do
},
sort: %{images: :desc}
},
%{page_size: 40},
Tag |> preload(:implied_tags)
%{page_size: 40}
)
|> Elasticsearch.search_records(preload(Tag, :implied_tags))
end
defp manages_links?(conn),