From 6bbe358dd1953884fc34ae5b4309f0a7c37e7460 Mon Sep 17 00:00:00 2001
From: liamwhite <liamwhite@users.noreply.github.com>
Date: Mon, 4 May 2020 08:19:42 -0400
Subject: [PATCH] Elasticsearch 7 upgrade (#113)

* initial upgrade to elasticsearch 7

* fix stat page error

* i am an idiot

* fix es not creating new indexes

* more complete removal of doc_type

Co-authored-by: Luna D <cod7777@yandex.ru>
---
 docker-compose.yml                            |   6 +-
 lib/philomena/comments/elasticsearch_index.ex |  38 +++---
 lib/philomena/elasticsearch.ex                |  22 ++-
 lib/philomena/elasticsearch_index.ex          |   6 -
 .../galleries/elasticsearch_index.ex          |  38 +++---
 lib/philomena/images/elasticsearch_index.ex   | 126 ++++++++----------
 lib/philomena/posts/elasticsearch_index.ex    |  48 +++----
 lib/philomena/reports/elasticsearch_index.ex  |  40 +++---
 lib/philomena/tags/elasticsearch_index.ex     |  56 ++++----
 .../templates/stat/index.html.slime           |   2 +-
 10 files changed, 164 insertions(+), 218 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index f0715c83..291eeacc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,17 +21,21 @@ services:
 
   postgres:
     image: postgres:12.2
+    environment:
+      - POSTGRES_PASSWORD=postgres
     volumes:
       - postgres_data:/var/lib/postgresql/data
     logging:
       driver: "none"
 
   elasticsearch:
-    image: elasticsearch:6.8.5
+    image: elasticsearch:7.6.2
     volumes:
       - elastic_data:/var/lib/elasticsearch
     logging:
       driver: "none"
+    environment:
+      - discovery.type=single-node
     ulimits:
       nofile:
           soft: 65536
diff --git a/lib/philomena/comments/elasticsearch_index.ex b/lib/philomena/comments/elasticsearch_index.ex
index 8e3ef4a7..54012b04 100644
--- a/lib/philomena/comments/elasticsearch_index.ex
+++ b/lib/philomena/comments/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do
     "comments"
   end
 
-  @impl true
-  def doc_type do
-    "comment"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,24 +16,21 @@ defmodule Philomena.Comments.ElasticsearchIndex do
         }
       },
       mappings: %{
-        comment: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            id: %{type: "integer"},
-            posted_at: %{type: "date"},
-            ip: %{type: "ip"},
-            fingerprint: %{type: "keyword"},
-            image_id: %{type: "keyword"},
-            user_id: %{type: "keyword"},
-            author: %{type: "keyword"},
-            image_tag_ids: %{type: "keyword"},
-            # boolean
-            anonymous: %{type: "keyword"},
-            # boolean
-            hidden_from_users: %{type: "keyword"},
-            body: %{type: "text", analyzer: "snowball"}
-          }
+        dynamic: false,
+        properties: %{
+          id: %{type: "integer"},
+          posted_at: %{type: "date"},
+          ip: %{type: "ip"},
+          fingerprint: %{type: "keyword"},
+          image_id: %{type: "keyword"},
+          user_id: %{type: "keyword"},
+          author: %{type: "keyword"},
+          image_tag_ids: %{type: "keyword"},
+          # boolean
+          anonymous: %{type: "keyword"},
+          # boolean
+          hidden_from_users: %{type: "keyword"},
+          body: %{type: "text", analyzer: "snowball"}
         }
       }
     }
diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex
index f4d92f81..4edd0c3b 100644
--- a/lib/philomena/elasticsearch.ex
+++ b/lib/philomena/elasticsearch.ex
@@ -50,14 +50,9 @@ defmodule Philomena.Elasticsearch do
     index = index_for(module)
 
     index_name = index.index_name()
-    doc_type = index.doc_type()
+    mapping = index.mapping().mappings.properties
 
-    mapping =
-      index.mapping().mappings
-      |> Map.get(String.to_atom(doc_type))
-      |> Map.get(:properties)
-
-    Elastix.Mapping.put(elastic_url(), index_name, doc_type, %{properties: mapping})
+    Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping})
   end
 
   def index_document(doc, module) do
@@ -67,7 +62,7 @@ defmodule Philomena.Elasticsearch do
     Elastix.Document.index(
       elastic_url(),
       index.index_name(),
-      [index.doc_type()],
+      "_doc",
       data.id,
       data
     )
@@ -79,7 +74,7 @@ defmodule Philomena.Elasticsearch do
     Elastix.Document.delete(
       elastic_url(),
       index.index_name(),
-      index.doc_type(),
+      "_doc",
       id
     )
   end
@@ -93,7 +88,7 @@ defmodule Philomena.Elasticsearch do
           doc = index.as_json(record)
 
           [
-            %{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}},
+            %{index: %{_index: index.index_name(), _id: doc.id}},
             doc
           ]
         end)
@@ -179,7 +174,7 @@ defmodule Philomena.Elasticsearch do
       Elastix.Search.search(
         elastic_url(),
         index.index_name(),
-        [index.doc_type()],
+        [],
         query_body
       )
 
@@ -194,12 +189,13 @@ defmodule Philomena.Elasticsearch do
       Map.merge(elastic_query, %{
         from: (page_number - 1) * page_size,
         size: page_size,
-        _source: false
+        _source: false,
+        track_total_hits: true
       })
 
     results = search(module, elastic_query)
     time = results["took"]
-    count = results["hits"]["total"]
+    count = results["hits"]["total"]["value"]
     entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"]))
 
     Logger.debug("[Elasticsearch] Query took #{time}ms")
diff --git a/lib/philomena/elasticsearch_index.ex b/lib/philomena/elasticsearch_index.ex
index e488359d..6810748e 100644
--- a/lib/philomena/elasticsearch_index.ex
+++ b/lib/philomena/elasticsearch_index.ex
@@ -3,12 +3,6 @@ defmodule Philomena.ElasticsearchIndex do
   # This is usually a collection name like "images".
   @callback index_name() :: String.t()
 
-  # Returns the document type for the index.
-  # This is usually the singular of the index name, like "image".
-  # 
-  # TODO: Remove for ES 7.0
-  @callback doc_type() :: String.t()
-
   # Returns the mapping and settings for the index.
   @callback mapping() :: map()
 
diff --git a/lib/philomena/galleries/elasticsearch_index.ex b/lib/philomena/galleries/elasticsearch_index.ex
index 3e047ba0..c2071b45 100644
--- a/lib/philomena/galleries/elasticsearch_index.ex
+++ b/lib/philomena/galleries/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
     "galleries"
   end
 
-  @impl true
-  def doc_type do
-    "gallery"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,24 +16,21 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
         }
       },
       mappings: %{
-        gallery: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            # keyword
-            id: %{type: "integer"},
-            image_count: %{type: "integer"},
-            watcher_count: %{type: "integer"},
-            updated_at: %{type: "date"},
-            created_at: %{type: "date"},
-            title: %{type: "keyword"},
-            # missing creator_id
-            creator: %{type: "keyword"},
-            image_ids: %{type: "keyword"},
-            # ???
-            watcher_ids: %{type: "keyword"},
-            description: %{type: "text", analyzer: "snowball"}
-          }
+        dynamic: false,
+        properties: %{
+          # keyword
+          id: %{type: "integer"},
+          image_count: %{type: "integer"},
+          watcher_count: %{type: "integer"},
+          updated_at: %{type: "date"},
+          created_at: %{type: "date"},
+          title: %{type: "keyword"},
+          # missing creator_id
+          creator: %{type: "keyword"},
+          image_ids: %{type: "keyword"},
+          # ???
+          watcher_ids: %{type: "keyword"},
+          description: %{type: "text", analyzer: "snowball"}
         }
       }
     }
diff --git a/lib/philomena/images/elasticsearch_index.ex b/lib/philomena/images/elasticsearch_index.ex
index d3a0c789..3e94e1a0 100644
--- a/lib/philomena/images/elasticsearch_index.ex
+++ b/lib/philomena/images/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Images.ElasticsearchIndex do
     "images"
   end
 
-  @impl true
-  def doc_type do
-    "image"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,68 +16,65 @@ defmodule Philomena.Images.ElasticsearchIndex do
         }
       },
       mappings: %{
-        image: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            anonymous: %{type: "boolean"},
-            aspect_ratio: %{type: "float"},
-            comment_count: %{type: "integer"},
-            commenters: %{type: "keyword"},
-            created_at: %{type: "date"},
-            deleted_by_user: %{type: "keyword"},
-            deleted_by_user_id: %{type: "keyword"},
-            deletion_reason: %{type: "text", analyzer: "snowball"},
-            description: %{type: "text", analyzer: "snowball"},
-            downvoter_ids: %{type: "keyword"},
-            downvoters: %{type: "keyword"},
-            downvotes: %{type: "integer"},
-            duplicate_id: %{type: "integer"},
-            faves: %{type: "integer"},
-            favourited_by_user_ids: %{type: "keyword"},
-            favourited_by_users: %{type: "keyword"},
-            file_name: %{type: "keyword"},
-            fingerprint: %{type: "keyword"},
-            first_seen_at: %{type: "date"},
-            height: %{type: "integer"},
-            hidden_by_user_ids: %{type: "keyword"},
-            hidden_by_users: %{type: "keyword"},
-            hidden_from_users: %{type: "keyword"},
-            id: %{type: "integer"},
-            ip: %{type: "ip"},
-            mime_type: %{type: "keyword"},
-            orig_sha512_hash: %{type: "keyword"},
-            original_format: %{type: "keyword"},
-            score: %{type: "integer"},
-            sha512_hash: %{type: "keyword"},
-            source_url: %{type: "keyword"},
-            tag_count: %{type: "integer"},
-            tag_ids: %{type: "keyword"},
-            tags: %{type: "text", analyzer: "keyword"},
-            true_uploader: %{type: "keyword"},
-            true_uploader_id: %{type: "keyword"},
-            updated_at: %{type: "date"},
-            uploader: %{type: "keyword"},
-            uploader_id: %{type: "keyword"},
-            upvoter_ids: %{type: "keyword"},
-            upvoters: %{type: "keyword"},
-            upvotes: %{type: "integer"},
-            user_id: %{type: "keyword"},
-            width: %{type: "integer"},
-            wilson_score: %{type: "float"},
-            galleries: %{
-              type: "nested",
-              properties: %{
-                id: %{type: "integer"},
-                position: %{type: "integer"}
-              }
-            },
-            namespaced_tags: %{
-              properties: %{
-                name: %{type: "keyword"},
-                name_in_namespace: %{type: "keyword"},
-                namespace: %{type: "keyword"}
-              }
+        dynamic: false,
+        properties: %{
+          anonymous: %{type: "boolean"},
+          aspect_ratio: %{type: "float"},
+          comment_count: %{type: "integer"},
+          commenters: %{type: "keyword"},
+          created_at: %{type: "date"},
+          deleted_by_user: %{type: "keyword"},
+          deleted_by_user_id: %{type: "keyword"},
+          deletion_reason: %{type: "text", analyzer: "snowball"},
+          description: %{type: "text", analyzer: "snowball"},
+          downvoter_ids: %{type: "keyword"},
+          downvoters: %{type: "keyword"},
+          downvotes: %{type: "integer"},
+          duplicate_id: %{type: "integer"},
+          faves: %{type: "integer"},
+          favourited_by_user_ids: %{type: "keyword"},
+          favourited_by_users: %{type: "keyword"},
+          file_name: %{type: "keyword"},
+          fingerprint: %{type: "keyword"},
+          first_seen_at: %{type: "date"},
+          height: %{type: "integer"},
+          hidden_by_user_ids: %{type: "keyword"},
+          hidden_by_users: %{type: "keyword"},
+          hidden_from_users: %{type: "keyword"},
+          id: %{type: "integer"},
+          ip: %{type: "ip"},
+          mime_type: %{type: "keyword"},
+          orig_sha512_hash: %{type: "keyword"},
+          original_format: %{type: "keyword"},
+          score: %{type: "integer"},
+          sha512_hash: %{type: "keyword"},
+          source_url: %{type: "keyword"},
+          tag_count: %{type: "integer"},
+          tag_ids: %{type: "keyword"},
+          tags: %{type: "text", analyzer: "keyword"},
+          true_uploader: %{type: "keyword"},
+          true_uploader_id: %{type: "keyword"},
+          updated_at: %{type: "date"},
+          uploader: %{type: "keyword"},
+          uploader_id: %{type: "keyword"},
+          upvoter_ids: %{type: "keyword"},
+          upvoters: %{type: "keyword"},
+          upvotes: %{type: "integer"},
+          user_id: %{type: "keyword"},
+          width: %{type: "integer"},
+          wilson_score: %{type: "float"},
+          galleries: %{
+            type: "nested",
+            properties: %{
+              id: %{type: "integer"},
+              position: %{type: "integer"}
+            }
+          },
+          namespaced_tags: %{
+            properties: %{
+              name: %{type: "keyword"},
+              name_in_namespace: %{type: "keyword"},
+              namespace: %{type: "keyword"}
             }
           }
         }
diff --git a/lib/philomena/posts/elasticsearch_index.ex b/lib/philomena/posts/elasticsearch_index.ex
index fb2a1bc2..0a165ed7 100644
--- a/lib/philomena/posts/elasticsearch_index.ex
+++ b/lib/philomena/posts/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do
     "posts"
   end
 
-  @impl true
-  def doc_type do
-    "post"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,29 +16,26 @@ defmodule Philomena.Posts.ElasticsearchIndex do
         }
       },
       mappings: %{
-        post: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            id: %{type: "integer"},
-            body: %{type: "text", analyzer: "snowball"},
-            ip: %{type: "ip"},
-            user_agent: %{type: "keyword"},
-            referrer: %{type: "keyword"},
-            fingerprint: %{type: "keyword"},
-            subject: %{type: "text", analyzer: "snowball"},
-            author: %{type: "keyword"},
-            topic_position: %{type: "integer"},
-            forum_id: %{type: "keyword"},
-            topic_id: %{type: "keyword"},
-            user_id: %{type: "keyword"},
-            anonymous: %{type: "boolean"},
-            updated_at: %{type: "date"},
-            created_at: %{type: "date"},
-            deleted: %{type: "boolean"},
-            access_level: %{type: "keyword"},
-            destroyed_content: %{type: "boolean"}
-          }
+        dynamic: false,
+        properties: %{
+          id: %{type: "integer"},
+          body: %{type: "text", analyzer: "snowball"},
+          ip: %{type: "ip"},
+          user_agent: %{type: "keyword"},
+          referrer: %{type: "keyword"},
+          fingerprint: %{type: "keyword"},
+          subject: %{type: "text", analyzer: "snowball"},
+          author: %{type: "keyword"},
+          topic_position: %{type: "integer"},
+          forum_id: %{type: "keyword"},
+          topic_id: %{type: "keyword"},
+          user_id: %{type: "keyword"},
+          anonymous: %{type: "boolean"},
+          updated_at: %{type: "date"},
+          created_at: %{type: "date"},
+          deleted: %{type: "boolean"},
+          access_level: %{type: "keyword"},
+          destroyed_content: %{type: "boolean"}
         }
       }
     }
diff --git a/lib/philomena/reports/elasticsearch_index.ex b/lib/philomena/reports/elasticsearch_index.ex
index f4f5ff1e..31f252e6 100644
--- a/lib/philomena/reports/elasticsearch_index.ex
+++ b/lib/philomena/reports/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do
     "reports"
   end
 
-  @impl true
-  def doc_type do
-    "report"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,25 +16,22 @@ defmodule Philomena.Reports.ElasticsearchIndex do
         }
       },
       mappings: %{
-        report: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            id: %{type: "integer"},
-            image_id: %{type: "integer"},
-            created_at: %{type: "date"},
-            ip: %{type: "ip"},
-            fingerprint: %{type: "keyword"},
-            state: %{type: "keyword"},
-            user: %{type: "keyword"},
-            user_id: %{type: "keyword"},
-            admin: %{type: "keyword"},
-            admin_id: %{type: "keyword"},
-            reportable_type: %{type: "keyword"},
-            reportable_id: %{type: "keyword"},
-            open: %{type: "boolean"},
-            reason: %{type: "text", analyzer: "snowball"}
-          }
+        dynamic: false,
+        properties: %{
+          id: %{type: "integer"},
+          image_id: %{type: "integer"},
+          created_at: %{type: "date"},
+          ip: %{type: "ip"},
+          fingerprint: %{type: "keyword"},
+          state: %{type: "keyword"},
+          user: %{type: "keyword"},
+          user_id: %{type: "keyword"},
+          admin: %{type: "keyword"},
+          admin_id: %{type: "keyword"},
+          reportable_type: %{type: "keyword"},
+          reportable_id: %{type: "keyword"},
+          open: %{type: "boolean"},
+          reason: %{type: "text", analyzer: "snowball"}
         }
       }
     }
diff --git a/lib/philomena/tags/elasticsearch_index.ex b/lib/philomena/tags/elasticsearch_index.ex
index ccea5b27..86a7e11e 100644
--- a/lib/philomena/tags/elasticsearch_index.ex
+++ b/lib/philomena/tags/elasticsearch_index.ex
@@ -6,11 +6,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do
     "tags"
   end
 
-  @impl true
-  def doc_type do
-    "tag"
-  end
-
   @impl true
   def mapping do
     %{
@@ -29,33 +24,30 @@ defmodule Philomena.Tags.ElasticsearchIndex do
         }
       },
       mappings: %{
-        tag: %{
-          _all: %{enabled: false},
-          dynamic: false,
-          properties: %{
-            id: %{type: "integer"},
-            images: %{type: "integer"},
-            slug: %{type: "keyword"},
-            name: %{type: "keyword"},
-            name_in_namespace: %{type: "keyword"},
-            namespace: %{type: "keyword"},
-            aliased_tag: %{type: "keyword"},
-            aliases: %{type: "keyword"},
-            implied_tags: %{type: "keyword"},
-            implied_tag_ids: %{type: "keyword"},
-            implied_by_tags: %{type: "keyword"},
-            category: %{type: "keyword"},
-            aliased: %{type: "boolean"},
-            analyzed_name: %{
-              type: "text",
-              fields: %{
-                nlp: %{type: "text", analyzer: "tag_snowball"},
-                ngram: %{type: "keyword"}
-              }
-            },
-            description: %{type: "text", analyzer: "snowball"},
-            short_description: %{type: "text", analyzer: "snowball"}
-          }
+        dynamic: false,
+        properties: %{
+          id: %{type: "integer"},
+          images: %{type: "integer"},
+          slug: %{type: "keyword"},
+          name: %{type: "keyword"},
+          name_in_namespace: %{type: "keyword"},
+          namespace: %{type: "keyword"},
+          aliased_tag: %{type: "keyword"},
+          aliases: %{type: "keyword"},
+          implied_tags: %{type: "keyword"},
+          implied_tag_ids: %{type: "keyword"},
+          implied_by_tags: %{type: "keyword"},
+          category: %{type: "keyword"},
+          aliased: %{type: "boolean"},
+          analyzed_name: %{
+            type: "text",
+            fields: %{
+              nlp: %{type: "text", analyzer: "tag_snowball"},
+              ngram: %{type: "keyword"}
+            }
+          },
+          description: %{type: "text", analyzer: "snowball"},
+          short_description: %{type: "text", analyzer: "snowball"}
         }
       }
     }
diff --git a/lib/philomena_web/templates/stat/index.html.slime b/lib/philomena_web/templates/stat/index.html.slime
index e066960c..fd5a9a55 100644
--- a/lib/philomena_web/templates/stat/index.html.slime
+++ b/lib/philomena_web/templates/stat/index.html.slime
@@ -23,7 +23,7 @@ elixir:
   p
     ' There are
     span.stat>
-      = number_with_delimiter(@comment_aggs["hits"]["total"])
+      = number_with_delimiter(@comment_aggs["hits"]["total"]["value"])
     ' comments on the site. Of these,
     => number_with_delimiter(cmt_bucket["deleted"]["doc_count"])
     ' have been deleted.