Elasticsearch 7 upgrade (#113)

* initial upgrade to elasticsearch 7

* fix stat page error

* i am an idiot

* fix es not creating new indexes

* more complete removal of doc_type

Co-authored-by: Luna D <cod7777@yandex.ru>

docker-compose.yml

@@ -21,17 +21,21 @@ services:
 
   postgres:
     image: postgres:12.2
+    environment:
+      - POSTGRES_PASSWORD=postgres
     volumes:
       - postgres_data:/var/lib/postgresql/data
     logging:
       driver: "none"
 
   elasticsearch:
-    image: elasticsearch:6.8.5
+    image: elasticsearch:7.6.2
     volumes:
       - elastic_data:/var/lib/elasticsearch
     logging:
       driver: "none"
+    environment:
+      - discovery.type=single-node
     ulimits:
       nofile:
           soft: 65536

lib/philomena/comments/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do
     "comments"
   end
 
-  @impl true
-  def doc_type do
-    "comment"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,8 +16,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do
         }
       },
       mappings: %{
-        comment: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
@@ -41,7 +34,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena/elasticsearch.ex

@@ -50,14 +50,9 @@ defmodule Philomena.Elasticsearch do
     index = index_for(module)
 
     index_name = index.index_name()
-    doc_type = index.doc_type()
+    mapping = index.mapping().mappings.properties
 
-    mapping =
-      index.mapping().mappings
-      |> Map.get(String.to_atom(doc_type))
-      |> Map.get(:properties)
-
-    Elastix.Mapping.put(elastic_url(), index_name, doc_type, %{properties: mapping})
+    Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping})
   end
 
   def index_document(doc, module) do
@@ -67,7 +62,7 @@ defmodule Philomena.Elasticsearch do
     Elastix.Document.index(
       elastic_url(),
       index.index_name(),
-      [index.doc_type()],
+      "_doc",
       data.id,
       data
     )
@@ -79,7 +74,7 @@ defmodule Philomena.Elasticsearch do
     Elastix.Document.delete(
       elastic_url(),
       index.index_name(),
-      index.doc_type(),
+      "_doc",
       id
     )
   end
@@ -93,7 +88,7 @@ defmodule Philomena.Elasticsearch do
           doc = index.as_json(record)
 
           [
-            %{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}},
+            %{index: %{_index: index.index_name(), _id: doc.id}},
             doc
           ]
         end)
@@ -179,7 +174,7 @@ defmodule Philomena.Elasticsearch do
       Elastix.Search.search(
         elastic_url(),
         index.index_name(),
-        [index.doc_type()],
+        [],
         query_body
       )
 
@@ -194,12 +189,13 @@ defmodule Philomena.Elasticsearch do
       Map.merge(elastic_query, %{
         from: (page_number - 1) * page_size,
         size: page_size,
-        _source: false
+        _source: false,
+        track_total_hits: true
       })
 
     results = search(module, elastic_query)
     time = results["took"]
-    count = results["hits"]["total"]
+    count = results["hits"]["total"]["value"]
     entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"]))
 
     Logger.debug("[Elasticsearch] Query took #{time}ms")

lib/philomena/elasticsearch_index.ex

@@ -3,12 +3,6 @@ defmodule Philomena.ElasticsearchIndex do
   # This is usually a collection name like "images".
   @callback index_name() :: String.t()
 
-  # Returns the document type for the index.
-  # This is usually the singular of the index name, like "image".
-  # 
-  # TODO: Remove for ES 7.0
-  @callback doc_type() :: String.t()
-
   # Returns the mapping and settings for the index.
   @callback mapping() :: map()
 

lib/philomena/galleries/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
     "galleries"
   end
 
-  @impl true
-  def doc_type do
-    "gallery"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,8 +16,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
         }
       },
       mappings: %{
-        gallery: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           # keyword
@@ -41,7 +34,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena/images/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Images.ElasticsearchIndex do
     "images"
   end
 
-  @impl true
-  def doc_type do
-    "image"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,8 +16,6 @@ defmodule Philomena.Images.ElasticsearchIndex do
         }
       },
       mappings: %{
-        image: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           anonymous: %{type: "boolean"},
@@ -87,7 +80,6 @@ defmodule Philomena.Images.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena/posts/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do
     "posts"
   end
 
-  @impl true
-  def doc_type do
-    "post"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,8 +16,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do
         }
       },
       mappings: %{
-        post: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
@@ -46,7 +39,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena/reports/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do
     "reports"
   end
 
-  @impl true
-  def doc_type do
-    "report"
-  end
-
   @impl true
   def mapping do
     %{
@@ -21,8 +16,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do
         }
       },
       mappings: %{
-        report: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
@@ -42,7 +35,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena/tags/elasticsearch_index.ex

@@ -6,11 +6,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do
     "tags"
   end
 
-  @impl true
-  def doc_type do
-    "tag"
-  end
-
   @impl true
   def mapping do
     %{
@@ -29,8 +24,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do
         }
       },
       mappings: %{
-        tag: %{
-          _all: %{enabled: false},
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
@@ -58,7 +51,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do
         }
       }
     }
-    }
   end
 
   @impl true

lib/philomena_web/templates/stat/index.html.slime

@@ -23,7 +23,7 @@ elixir:
   p
     ' There are
     span.stat>
-      = number_with_delimiter(@comment_aggs["hits"]["total"])
+      = number_with_delimiter(@comment_aggs["hits"]["total"]["value"])
     ' comments on the site. Of these,
     => number_with_delimiter(cmt_bucket["deleted"]["doc_count"])
     ' have been deleted.