diff --git a/docker-compose.yml b/docker-compose.yml index f0715c83..291eeacc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,17 +21,21 @@ services: postgres: image: postgres:12.2 + environment: + - POSTGRES_PASSWORD=postgres volumes: - postgres_data:/var/lib/postgresql/data logging: driver: "none" elasticsearch: - image: elasticsearch:6.8.5 + image: elasticsearch:7.6.2 volumes: - elastic_data:/var/lib/elasticsearch logging: driver: "none" + environment: + - discovery.type=single-node ulimits: nofile: soft: 65536 diff --git a/lib/philomena/comments/elasticsearch_index.ex b/lib/philomena/comments/elasticsearch_index.ex index 8e3ef4a7..54012b04 100644 --- a/lib/philomena/comments/elasticsearch_index.ex +++ b/lib/philomena/comments/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do "comments" end - @impl true - def doc_type do - "comment" - end - @impl true def mapping do %{ @@ -21,24 +16,21 @@ defmodule Philomena.Comments.ElasticsearchIndex do } }, mappings: %{ - comment: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - id: %{type: "integer"}, - posted_at: %{type: "date"}, - ip: %{type: "ip"}, - fingerprint: %{type: "keyword"}, - image_id: %{type: "keyword"}, - user_id: %{type: "keyword"}, - author: %{type: "keyword"}, - image_tag_ids: %{type: "keyword"}, - # boolean - anonymous: %{type: "keyword"}, - # boolean - hidden_from_users: %{type: "keyword"}, - body: %{type: "text", analyzer: "snowball"} - } + dynamic: false, + properties: %{ + id: %{type: "integer"}, + posted_at: %{type: "date"}, + ip: %{type: "ip"}, + fingerprint: %{type: "keyword"}, + image_id: %{type: "keyword"}, + user_id: %{type: "keyword"}, + author: %{type: "keyword"}, + image_tag_ids: %{type: "keyword"}, + # boolean + anonymous: %{type: "keyword"}, + # boolean + hidden_from_users: %{type: "keyword"}, + body: %{type: "text", analyzer: "snowball"} } } } diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex index f4d92f81..4edd0c3b 100644 --- a/lib/philomena/elasticsearch.ex +++ b/lib/philomena/elasticsearch.ex @@ -50,14 +50,9 @@ defmodule Philomena.Elasticsearch do index = index_for(module) index_name = index.index_name() - doc_type = index.doc_type() + mapping = index.mapping().mappings.properties - mapping = - index.mapping().mappings - |> Map.get(String.to_atom(doc_type)) - |> Map.get(:properties) - - Elastix.Mapping.put(elastic_url(), index_name, doc_type, %{properties: mapping}) + Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping}) end def index_document(doc, module) do @@ -67,7 +62,7 @@ defmodule Philomena.Elasticsearch do Elastix.Document.index( elastic_url(), index.index_name(), - [index.doc_type()], + "_doc", data.id, data ) @@ -79,7 +74,7 @@ defmodule Philomena.Elasticsearch do Elastix.Document.delete( elastic_url(), index.index_name(), - index.doc_type(), + "_doc", id ) end @@ -93,7 +88,7 @@ defmodule Philomena.Elasticsearch do doc = index.as_json(record) [ - %{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}}, + %{index: %{_index: index.index_name(), _id: doc.id}}, doc ] end) @@ -179,7 +174,7 @@ defmodule Philomena.Elasticsearch do Elastix.Search.search( elastic_url(), index.index_name(), - [index.doc_type()], + [], query_body ) @@ -194,12 +189,13 @@ defmodule Philomena.Elasticsearch do Map.merge(elastic_query, %{ from: (page_number - 1) * page_size, size: page_size, - _source: false + _source: false, + track_total_hits: true }) results = search(module, elastic_query) time = results["took"] - count = results["hits"]["total"] + count = results["hits"]["total"]["value"] entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"])) Logger.debug("[Elasticsearch] Query took #{time}ms") diff --git a/lib/philomena/elasticsearch_index.ex b/lib/philomena/elasticsearch_index.ex index e488359d..6810748e 100644 --- a/lib/philomena/elasticsearch_index.ex +++ b/lib/philomena/elasticsearch_index.ex @@ -3,12 +3,6 @@ defmodule Philomena.ElasticsearchIndex do # This is usually a collection name like "images". @callback index_name() :: String.t() - # Returns the document type for the index. - # This is usually the singular of the index name, like "image". - # - # TODO: Remove for ES 7.0 - @callback doc_type() :: String.t() - # Returns the mapping and settings for the index. @callback mapping() :: map() diff --git a/lib/philomena/galleries/elasticsearch_index.ex b/lib/philomena/galleries/elasticsearch_index.ex index 3e047ba0..c2071b45 100644 --- a/lib/philomena/galleries/elasticsearch_index.ex +++ b/lib/philomena/galleries/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do "galleries" end - @impl true - def doc_type do - "gallery" - end - @impl true def mapping do %{ @@ -21,24 +16,21 @@ defmodule Philomena.Galleries.ElasticsearchIndex do } }, mappings: %{ - gallery: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - # keyword - id: %{type: "integer"}, - image_count: %{type: "integer"}, - watcher_count: %{type: "integer"}, - updated_at: %{type: "date"}, - created_at: %{type: "date"}, - title: %{type: "keyword"}, - # missing creator_id - creator: %{type: "keyword"}, - image_ids: %{type: "keyword"}, - # ??? - watcher_ids: %{type: "keyword"}, - description: %{type: "text", analyzer: "snowball"} - } + dynamic: false, + properties: %{ + # keyword + id: %{type: "integer"}, + image_count: %{type: "integer"}, + watcher_count: %{type: "integer"}, + updated_at: %{type: "date"}, + created_at: %{type: "date"}, + title: %{type: "keyword"}, + # missing creator_id + creator: %{type: "keyword"}, + image_ids: %{type: "keyword"}, + # ??? + watcher_ids: %{type: "keyword"}, + description: %{type: "text", analyzer: "snowball"} } } } diff --git a/lib/philomena/images/elasticsearch_index.ex b/lib/philomena/images/elasticsearch_index.ex index d3a0c789..3e94e1a0 100644 --- a/lib/philomena/images/elasticsearch_index.ex +++ b/lib/philomena/images/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Images.ElasticsearchIndex do "images" end - @impl true - def doc_type do - "image" - end - @impl true def mapping do %{ @@ -21,68 +16,65 @@ defmodule Philomena.Images.ElasticsearchIndex do } }, mappings: %{ - image: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - anonymous: %{type: "boolean"}, - aspect_ratio: %{type: "float"}, - comment_count: %{type: "integer"}, - commenters: %{type: "keyword"}, - created_at: %{type: "date"}, - deleted_by_user: %{type: "keyword"}, - deleted_by_user_id: %{type: "keyword"}, - deletion_reason: %{type: "text", analyzer: "snowball"}, - description: %{type: "text", analyzer: "snowball"}, - downvoter_ids: %{type: "keyword"}, - downvoters: %{type: "keyword"}, - downvotes: %{type: "integer"}, - duplicate_id: %{type: "integer"}, - faves: %{type: "integer"}, - favourited_by_user_ids: %{type: "keyword"}, - favourited_by_users: %{type: "keyword"}, - file_name: %{type: "keyword"}, - fingerprint: %{type: "keyword"}, - first_seen_at: %{type: "date"}, - height: %{type: "integer"}, - hidden_by_user_ids: %{type: "keyword"}, - hidden_by_users: %{type: "keyword"}, - hidden_from_users: %{type: "keyword"}, - id: %{type: "integer"}, - ip: %{type: "ip"}, - mime_type: %{type: "keyword"}, - orig_sha512_hash: %{type: "keyword"}, - original_format: %{type: "keyword"}, - score: %{type: "integer"}, - sha512_hash: %{type: "keyword"}, - source_url: %{type: "keyword"}, - tag_count: %{type: "integer"}, - tag_ids: %{type: "keyword"}, - tags: %{type: "text", analyzer: "keyword"}, - true_uploader: %{type: "keyword"}, - true_uploader_id: %{type: "keyword"}, - updated_at: %{type: "date"}, - uploader: %{type: "keyword"}, - uploader_id: %{type: "keyword"}, - upvoter_ids: %{type: "keyword"}, - upvoters: %{type: "keyword"}, - upvotes: %{type: "integer"}, - user_id: %{type: "keyword"}, - width: %{type: "integer"}, - wilson_score: %{type: "float"}, - galleries: %{ - type: "nested", - properties: %{ - id: %{type: "integer"}, - position: %{type: "integer"} - } - }, - namespaced_tags: %{ - properties: %{ - name: %{type: "keyword"}, - name_in_namespace: %{type: "keyword"}, - namespace: %{type: "keyword"} - } + dynamic: false, + properties: %{ + anonymous: %{type: "boolean"}, + aspect_ratio: %{type: "float"}, + comment_count: %{type: "integer"}, + commenters: %{type: "keyword"}, + created_at: %{type: "date"}, + deleted_by_user: %{type: "keyword"}, + deleted_by_user_id: %{type: "keyword"}, + deletion_reason: %{type: "text", analyzer: "snowball"}, + description: %{type: "text", analyzer: "snowball"}, + downvoter_ids: %{type: "keyword"}, + downvoters: %{type: "keyword"}, + downvotes: %{type: "integer"}, + duplicate_id: %{type: "integer"}, + faves: %{type: "integer"}, + favourited_by_user_ids: %{type: "keyword"}, + favourited_by_users: %{type: "keyword"}, + file_name: %{type: "keyword"}, + fingerprint: %{type: "keyword"}, + first_seen_at: %{type: "date"}, + height: %{type: "integer"}, + hidden_by_user_ids: %{type: "keyword"}, + hidden_by_users: %{type: "keyword"}, + hidden_from_users: %{type: "keyword"}, + id: %{type: "integer"}, + ip: %{type: "ip"}, + mime_type: %{type: "keyword"}, + orig_sha512_hash: %{type: "keyword"}, + original_format: %{type: "keyword"}, + score: %{type: "integer"}, + sha512_hash: %{type: "keyword"}, + source_url: %{type: "keyword"}, + tag_count: %{type: "integer"}, + tag_ids: %{type: "keyword"}, + tags: %{type: "text", analyzer: "keyword"}, + true_uploader: %{type: "keyword"}, + true_uploader_id: %{type: "keyword"}, + updated_at: %{type: "date"}, + uploader: %{type: "keyword"}, + uploader_id: %{type: "keyword"}, + upvoter_ids: %{type: "keyword"}, + upvoters: %{type: "keyword"}, + upvotes: %{type: "integer"}, + user_id: %{type: "keyword"}, + width: %{type: "integer"}, + wilson_score: %{type: "float"}, + galleries: %{ + type: "nested", + properties: %{ + id: %{type: "integer"}, + position: %{type: "integer"} + } + }, + namespaced_tags: %{ + properties: %{ + name: %{type: "keyword"}, + name_in_namespace: %{type: "keyword"}, + namespace: %{type: "keyword"} } } } diff --git a/lib/philomena/posts/elasticsearch_index.ex b/lib/philomena/posts/elasticsearch_index.ex index fb2a1bc2..0a165ed7 100644 --- a/lib/philomena/posts/elasticsearch_index.ex +++ b/lib/philomena/posts/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do "posts" end - @impl true - def doc_type do - "post" - end - @impl true def mapping do %{ @@ -21,29 +16,26 @@ defmodule Philomena.Posts.ElasticsearchIndex do } }, mappings: %{ - post: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - id: %{type: "integer"}, - body: %{type: "text", analyzer: "snowball"}, - ip: %{type: "ip"}, - user_agent: %{type: "keyword"}, - referrer: %{type: "keyword"}, - fingerprint: %{type: "keyword"}, - subject: %{type: "text", analyzer: "snowball"}, - author: %{type: "keyword"}, - topic_position: %{type: "integer"}, - forum_id: %{type: "keyword"}, - topic_id: %{type: "keyword"}, - user_id: %{type: "keyword"}, - anonymous: %{type: "boolean"}, - updated_at: %{type: "date"}, - created_at: %{type: "date"}, - deleted: %{type: "boolean"}, - access_level: %{type: "keyword"}, - destroyed_content: %{type: "boolean"} - } + dynamic: false, + properties: %{ + id: %{type: "integer"}, + body: %{type: "text", analyzer: "snowball"}, + ip: %{type: "ip"}, + user_agent: %{type: "keyword"}, + referrer: %{type: "keyword"}, + fingerprint: %{type: "keyword"}, + subject: %{type: "text", analyzer: "snowball"}, + author: %{type: "keyword"}, + topic_position: %{type: "integer"}, + forum_id: %{type: "keyword"}, + topic_id: %{type: "keyword"}, + user_id: %{type: "keyword"}, + anonymous: %{type: "boolean"}, + updated_at: %{type: "date"}, + created_at: %{type: "date"}, + deleted: %{type: "boolean"}, + access_level: %{type: "keyword"}, + destroyed_content: %{type: "boolean"} } } } diff --git a/lib/philomena/reports/elasticsearch_index.ex b/lib/philomena/reports/elasticsearch_index.ex index f4f5ff1e..31f252e6 100644 --- a/lib/philomena/reports/elasticsearch_index.ex +++ b/lib/philomena/reports/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do "reports" end - @impl true - def doc_type do - "report" - end - @impl true def mapping do %{ @@ -21,25 +16,22 @@ defmodule Philomena.Reports.ElasticsearchIndex do } }, mappings: %{ - report: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - id: %{type: "integer"}, - image_id: %{type: "integer"}, - created_at: %{type: "date"}, - ip: %{type: "ip"}, - fingerprint: %{type: "keyword"}, - state: %{type: "keyword"}, - user: %{type: "keyword"}, - user_id: %{type: "keyword"}, - admin: %{type: "keyword"}, - admin_id: %{type: "keyword"}, - reportable_type: %{type: "keyword"}, - reportable_id: %{type: "keyword"}, - open: %{type: "boolean"}, - reason: %{type: "text", analyzer: "snowball"} - } + dynamic: false, + properties: %{ + id: %{type: "integer"}, + image_id: %{type: "integer"}, + created_at: %{type: "date"}, + ip: %{type: "ip"}, + fingerprint: %{type: "keyword"}, + state: %{type: "keyword"}, + user: %{type: "keyword"}, + user_id: %{type: "keyword"}, + admin: %{type: "keyword"}, + admin_id: %{type: "keyword"}, + reportable_type: %{type: "keyword"}, + reportable_id: %{type: "keyword"}, + open: %{type: "boolean"}, + reason: %{type: "text", analyzer: "snowball"} } } } diff --git a/lib/philomena/tags/elasticsearch_index.ex b/lib/philomena/tags/elasticsearch_index.ex index ccea5b27..86a7e11e 100644 --- a/lib/philomena/tags/elasticsearch_index.ex +++ b/lib/philomena/tags/elasticsearch_index.ex @@ -6,11 +6,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do "tags" end - @impl true - def doc_type do - "tag" - end - @impl true def mapping do %{ @@ -29,33 +24,30 @@ defmodule Philomena.Tags.ElasticsearchIndex do } }, mappings: %{ - tag: %{ - _all: %{enabled: false}, - dynamic: false, - properties: %{ - id: %{type: "integer"}, - images: %{type: "integer"}, - slug: %{type: "keyword"}, - name: %{type: "keyword"}, - name_in_namespace: %{type: "keyword"}, - namespace: %{type: "keyword"}, - aliased_tag: %{type: "keyword"}, - aliases: %{type: "keyword"}, - implied_tags: %{type: "keyword"}, - implied_tag_ids: %{type: "keyword"}, - implied_by_tags: %{type: "keyword"}, - category: %{type: "keyword"}, - aliased: %{type: "boolean"}, - analyzed_name: %{ - type: "text", - fields: %{ - nlp: %{type: "text", analyzer: "tag_snowball"}, - ngram: %{type: "keyword"} - } - }, - description: %{type: "text", analyzer: "snowball"}, - short_description: %{type: "text", analyzer: "snowball"} - } + dynamic: false, + properties: %{ + id: %{type: "integer"}, + images: %{type: "integer"}, + slug: %{type: "keyword"}, + name: %{type: "keyword"}, + name_in_namespace: %{type: "keyword"}, + namespace: %{type: "keyword"}, + aliased_tag: %{type: "keyword"}, + aliases: %{type: "keyword"}, + implied_tags: %{type: "keyword"}, + implied_tag_ids: %{type: "keyword"}, + implied_by_tags: %{type: "keyword"}, + category: %{type: "keyword"}, + aliased: %{type: "boolean"}, + analyzed_name: %{ + type: "text", + fields: %{ + nlp: %{type: "text", analyzer: "tag_snowball"}, + ngram: %{type: "keyword"} + } + }, + description: %{type: "text", analyzer: "snowball"}, + short_description: %{type: "text", analyzer: "snowball"} } } } diff --git a/lib/philomena_web/templates/stat/index.html.slime b/lib/philomena_web/templates/stat/index.html.slime index e066960c..fd5a9a55 100644 --- a/lib/philomena_web/templates/stat/index.html.slime +++ b/lib/philomena_web/templates/stat/index.html.slime @@ -23,7 +23,7 @@ elixir: p ' There are span.stat> - = number_with_delimiter(@comment_aggs["hits"]["total"]) + = number_with_delimiter(@comment_aggs["hits"]["total"]["value"]) ' comments on the site. Of these, => number_with_delimiter(cmt_bucket["deleted"]["doc_count"]) ' have been deleted.