From 19c5ded879af0443c7697bfa6508f87464d7432d Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Thu, 3 Oct 2019 20:42:49 -0400 Subject: [PATCH] add tag searching --- lib/philomena/images/query.ex | 1 - lib/philomena/search/parser.ex | 4 +- lib/philomena/tags/elasticsearch.ex | 74 +++++++++++++++++++ lib/philomena/tags/query.ex | 22 ++++++ lib/philomena/tags/tag.ex | 15 +++- .../controllers/tag_controller.ex | 17 ++++- 6 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 lib/philomena/tags/elasticsearch.ex create mode 100644 lib/philomena/tags/query.ex diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index d1abd431..aebd8743 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -1,7 +1,6 @@ defmodule Philomena.Images.Query do import Philomena.Search.Parser import Philomena.Search.String - alias Philomena.Repo defparser("anonymous", int: diff --git a/lib/philomena/search/parser.ex b/lib/philomena/search/parser.ex index 271a8a33..80e8062e 100644 --- a/lib/philomena/search/parser.ex +++ b/lib/philomena/search/parser.ex @@ -1,7 +1,7 @@ defmodule Philomena.Search.Parser do defmacro defparser(name, opts) do - field_transforms = Keyword.get(opts, :transforms, %{}) - field_aliases = Keyword.get(opts, :aliases, %{}) + field_transforms = Keyword.get(opts, :transforms, %{}) |> Macro.escape + field_aliases = Keyword.get(opts, :aliases, %{}) |> Macro.escape default_field = Keyword.fetch!(opts, :default) quote location: :keep do diff --git a/lib/philomena/tags/elasticsearch.ex b/lib/philomena/tags/elasticsearch.ex new file mode 100644 index 00000000..7cce7184 --- /dev/null +++ b/lib/philomena/tags/elasticsearch.ex @@ -0,0 +1,74 @@ +defmodule Philomena.Tags.Elasticsearch do + def mapping do + %{ + settings: %{ + index: %{ + number_of_shards: 5, + max_result_window: 10_000_000, + analysis: %{ + analyzer: %{ + tag_snowball: %{ + tokenizer: :letter, + filter: [:asciifolding, :snowball] + } + } + } + } + }, + mappings: %{ + tag: %{ + _all: %{enabled: false}, + dynamic: false, + properties: %{ + id: %{type: "integer"}, + images: %{type: "integer"}, + slug: %{type: "keyword"}, + name: %{type: "keyword"}, + name_in_namespace: %{type: "keyword"}, + namespace: %{type: "keyword"}, + aliased_tag: %{type: "keyword"}, + aliases: %{type: "keyword"}, + implied_tags: %{type: "keyword"}, + implied_tag_ids: %{type: "keyword"}, + implied_by_tags: %{type: "keyword"}, + category: %{type: "keyword"}, + aliased: %{type: "boolean"}, + analyzed_name: %{ + type: "text", + fields: %{ + nlp: %{type: "text", analyzer: "tag_snowball"}, + ngram: %{type: "keyword"} + } + }, + description: %{type: "text", analyzer: "snowball"}, + short_description: %{type: "text", analyzer: "snowball"} + } + } + } + } + end + + # preload([ + # :aliased_tag, :aliases, :implied_tags, :implied_by_tags + # ]) + def as_json(tag) do + %{ + id: tag.id, + images: tag.images_count, + slug: tag.slug, + name: tag.name, + name_in_namespace: tag.name_in_namespace, + namespace: tag.namespace, + analyzed_name: tag.name, + implied_tags: tag.implied_tags |> Enum.map(& &1.name), + implied_tag_ids: tag.implied_tags |> Enum.map(& &1.id), + implied_by_tags: tag.implied_by_tags |> Enum.map(& &1.name), + aliased_tag: if(!!tag.aliased_tag, do: tag.aliased_tag.name), + aliases: tag.aliases |> Enum.map(& &1.name), + category: tag.category, + aliased: !!tag.aliased_tag, + description: tag.description, + short_description: tag.description + } + end +end diff --git a/lib/philomena/tags/query.ex b/lib/philomena/tags/query.ex new file mode 100644 index 00000000..3403495d --- /dev/null +++ b/lib/philomena/tags/query.ex @@ -0,0 +1,22 @@ +defmodule Philomena.Tags.Query do + import Philomena.Search.Parser + + defparser("tag", + int: ~W(id images), + literal: ~W(slug name name_in_namespace namespace implies alias_of implied_by aliases category analyzed_name), + boolean: ~W(aliased), + ngram: ~W(description short_description), + aliases: %{ + "implies" => "implied_tags", + "implied_by" => "implied_by_tags", + "alias_of" => "aliased_tag" + }, + default: "analyzed_name" + ) + + def compile(query_string) do + query_string = query_string || "" + + tag_parser(%{}, query_string) + end +end diff --git a/lib/philomena/tags/tag.ex b/lib/philomena/tags/tag.ex index 6ebe122f..6e764a56 100644 --- a/lib/philomena/tags/tag.ex +++ b/lib/philomena/tags/tag.ex @@ -2,16 +2,29 @@ defmodule Philomena.Tags.Tag do use Ecto.Schema import Ecto.Changeset + use Philomena.Elasticsearch, + definition: Philomena.Tags.Elasticsearch, + index_name: "tags", + doc_type: "tag" + schema "tags" do belongs_to :aliased_tag, Philomena.Tags.Tag, source: :aliased_tag_id has_many :aliases, Philomena.Tags.Tag, foreign_key: :aliased_tag_id + many_to_many :implied_tags, Philomena.Tags.Tag, join_through: "tags_implied_tags", join_keys: [tag_id: :id, implied_tag_id: :id] + many_to_many :implied_by_tags, Philomena.Tags.Tag, join_through: "tags_implied_tags", join_keys: [implied_tag_id: :id, tag_id: :id] field :slug, :string field :name, :string field :category, :string - field :images_count, :integer + field :images_count, :integer, default: 0 field :description, :string field :short_description, :string + field :namespace, :string + field :name_in_namespace, :string + field :image, :string + field :image_format, :string + field :image_mime_type, :string + field :mod_notes, :string timestamps(inserted_at: :created_at) end diff --git a/lib/philomena_web/controllers/tag_controller.ex b/lib/philomena_web/controllers/tag_controller.ex index 18a2c3b7..ece119fb 100644 --- a/lib/philomena_web/controllers/tag_controller.ex +++ b/lib/philomena_web/controllers/tag_controller.ex @@ -1,13 +1,24 @@ defmodule PhilomenaWeb.TagController do use PhilomenaWeb, :controller - alias Philomena.{Images.Image, Tags} + alias Philomena.{Images.Image, Tags, Tags.Tag} import Ecto.Query plug ImageFilter - def index(conn, _params) do - tags = Tags.list_tags() + def index(conn, params) do + {:ok, query} = Tags.Query.compile(params["tq"] || "*") + + tags = + Tag.search_records( + %{ + query: query, + size: 250, + sort: [%{images: :desc}, %{name: :asc}] + }, + Tag + ) + render(conn, "index.html", tags: tags) end