PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2024-11-23 20:18:00 +01:00
Code Issues Projects Releases Packages Wiki Activity

Elasticsearch 7 upgrade (#113)

Browse source 
* initial upgrade to elasticsearch 7

* fix stat page error

* i am an idiot

* fix es not creating new indexes

* more complete removal of doc_type

Co-authored-by: Luna D <cod7777@yandex.ru>
This commit is contained in:
liamwhite 2020-05-04 08:19:42 -04:00 committed by GitHub
parent 4b86e783ef
commit 6bbe358dd1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 164 additions and 218 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
docker-compose.yml
View file

@ -21,17 +21,21 @@ services:
postgres: postgres:
image: postgres:12.2 image: postgres:12.2
environment:
- POSTGRES_PASSWORD=postgres
volumes: volumes:
- postgres_data:/var/lib/postgresql/data - postgres_data:/var/lib/postgresql/data
logging: logging:
driver: "none" driver: "none"
elasticsearch: elasticsearch:
image: elasticsearch:6.8.5 image: elasticsearch:7.6.2
volumes: volumes:
- elastic_data:/var/lib/elasticsearch - elastic_data:/var/lib/elasticsearch
logging: logging:
driver: "none" driver: "none"
environment:
- discovery.type=single-node
ulimits: ulimits:
nofile: nofile:
soft: 65536 soft: 65536

38
lib/philomena/comments/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Comments.ElasticsearchIndex do
"comments" "comments"
end end
@impl true
def doc_type do
"comment"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -21,24 +16,21 @@ defmodule Philomena.Comments.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
comment: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, id: %{type: "integer"},
properties: %{ posted_at: %{type: "date"},
id: %{type: "integer"}, ip: %{type: "ip"},
posted_at: %{type: "date"}, fingerprint: %{type: "keyword"},
ip: %{type: "ip"}, image_id: %{type: "keyword"},
fingerprint: %{type: "keyword"}, user_id: %{type: "keyword"},
image_id: %{type: "keyword"}, author: %{type: "keyword"},
user_id: %{type: "keyword"}, image_tag_ids: %{type: "keyword"},
author: %{type: "keyword"}, # boolean
image_tag_ids: %{type: "keyword"}, anonymous: %{type: "keyword"},
# boolean # boolean
anonymous: %{type: "keyword"}, hidden_from_users: %{type: "keyword"},
# boolean body: %{type: "text", analyzer: "snowball"}
hidden_from_users: %{type: "keyword"},
body: %{type: "text", analyzer: "snowball"}
}
} }
} }
} }

22
lib/philomena/elasticsearch.ex
View file

@ -50,14 +50,9 @@ defmodule Philomena.Elasticsearch do
index = index_for(module) index = index_for(module)
index_name = index.index_name() index_name = index.index_name()
doc_type = index.doc_type() mapping = index.mapping().mappings.properties
mapping = Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping})
index.mapping().mappings
|> Map.get(String.to_atom(doc_type))
|> Map.get(:properties)
Elastix.Mapping.put(elastic_url(), index_name, doc_type, %{properties: mapping})
end end
def index_document(doc, module) do def index_document(doc, module) do
@ -67,7 +62,7 @@ defmodule Philomena.Elasticsearch do
Elastix.Document.index( Elastix.Document.index(
elastic_url(), elastic_url(),
index.index_name(), index.index_name(),
[index.doc_type()], "_doc",
data.id, data.id,
data data
) )
@ -79,7 +74,7 @@ defmodule Philomena.Elasticsearch do
Elastix.Document.delete( Elastix.Document.delete(
elastic_url(), elastic_url(),
index.index_name(), index.index_name(),
index.doc_type(), "_doc",
id id
) )
end end
@ -93,7 +88,7 @@ defmodule Philomena.Elasticsearch do
doc = index.as_json(record) doc = index.as_json(record)
[ [
%{index: %{_index: index.index_name(), _type: index.doc_type(), _id: doc.id}}, %{index: %{_index: index.index_name(), _id: doc.id}},
doc doc
] ]
end) end)
@ -179,7 +174,7 @@ defmodule Philomena.Elasticsearch do
Elastix.Search.search( Elastix.Search.search(
elastic_url(), elastic_url(),
index.index_name(), index.index_name(),
[index.doc_type()], [],
query_body query_body
) )
@ -194,12 +189,13 @@ defmodule Philomena.Elasticsearch do
Map.merge(elastic_query, %{ Map.merge(elastic_query, %{
from: (page_number - 1) * page_size, from: (page_number - 1) * page_size,
size: page_size, size: page_size,
_source: false _source: false,
track_total_hits: true
}) })
results = search(module, elastic_query) results = search(module, elastic_query)
time = results["took"] time = results["took"]
count = results["hits"]["total"] count = results["hits"]["total"]["value"]
entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"])) entries = Enum.map(results["hits"]["hits"], &String.to_integer(&1["_id"]))
Logger.debug("[Elasticsearch] Query took #{time}ms") Logger.debug("[Elasticsearch] Query took #{time}ms")

6
lib/philomena/elasticsearch_index.ex
View file

@ -3,12 +3,6 @@ defmodule Philomena.ElasticsearchIndex do
# This is usually a collection name like "images". # This is usually a collection name like "images".
@callback index_name() :: String.t() @callback index_name() :: String.t()
# Returns the document type for the index.
# This is usually the singular of the index name, like "image".
#
# TODO: Remove for ES 7.0
@callback doc_type() :: String.t()
# Returns the mapping and settings for the index. # Returns the mapping and settings for the index.
@callback mapping() :: map() @callback mapping() :: map()

38
lib/philomena/galleries/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
"galleries" "galleries"
end end
@impl true
def doc_type do
"gallery"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -21,24 +16,21 @@ defmodule Philomena.Galleries.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
gallery: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, # keyword
properties: %{ id: %{type: "integer"},
# keyword image_count: %{type: "integer"},
id: %{type: "integer"}, watcher_count: %{type: "integer"},
image_count: %{type: "integer"}, updated_at: %{type: "date"},
watcher_count: %{type: "integer"}, created_at: %{type: "date"},
updated_at: %{type: "date"}, title: %{type: "keyword"},
created_at: %{type: "date"}, # missing creator_id
title: %{type: "keyword"}, creator: %{type: "keyword"},
# missing creator_id image_ids: %{type: "keyword"},
creator: %{type: "keyword"}, # ???
image_ids: %{type: "keyword"}, watcher_ids: %{type: "keyword"},
# ??? description: %{type: "text", analyzer: "snowball"}
watcher_ids: %{type: "keyword"},
description: %{type: "text", analyzer: "snowball"}
}
} }
} }
} }

126
lib/philomena/images/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Images.ElasticsearchIndex do
"images" "images"
end end
@impl true
def doc_type do
"image"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -21,68 +16,65 @@ defmodule Philomena.Images.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
image: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, anonymous: %{type: "boolean"},
properties: %{ aspect_ratio: %{type: "float"},
anonymous: %{type: "boolean"}, comment_count: %{type: "integer"},
aspect_ratio: %{type: "float"}, commenters: %{type: "keyword"},
comment_count: %{type: "integer"}, created_at: %{type: "date"},
commenters: %{type: "keyword"}, deleted_by_user: %{type: "keyword"},
created_at: %{type: "date"}, deleted_by_user_id: %{type: "keyword"},
deleted_by_user: %{type: "keyword"}, deletion_reason: %{type: "text", analyzer: "snowball"},
deleted_by_user_id: %{type: "keyword"}, description: %{type: "text", analyzer: "snowball"},
deletion_reason: %{type: "text", analyzer: "snowball"}, downvoter_ids: %{type: "keyword"},
description: %{type: "text", analyzer: "snowball"}, downvoters: %{type: "keyword"},
downvoter_ids: %{type: "keyword"}, downvotes: %{type: "integer"},
downvoters: %{type: "keyword"}, duplicate_id: %{type: "integer"},
downvotes: %{type: "integer"}, faves: %{type: "integer"},
duplicate_id: %{type: "integer"}, favourited_by_user_ids: %{type: "keyword"},
faves: %{type: "integer"}, favourited_by_users: %{type: "keyword"},
favourited_by_user_ids: %{type: "keyword"}, file_name: %{type: "keyword"},
favourited_by_users: %{type: "keyword"}, fingerprint: %{type: "keyword"},
file_name: %{type: "keyword"}, first_seen_at: %{type: "date"},
fingerprint: %{type: "keyword"}, height: %{type: "integer"},
first_seen_at: %{type: "date"}, hidden_by_user_ids: %{type: "keyword"},
height: %{type: "integer"}, hidden_by_users: %{type: "keyword"},
hidden_by_user_ids: %{type: "keyword"}, hidden_from_users: %{type: "keyword"},
hidden_by_users: %{type: "keyword"}, id: %{type: "integer"},
hidden_from_users: %{type: "keyword"}, ip: %{type: "ip"},
id: %{type: "integer"}, mime_type: %{type: "keyword"},
ip: %{type: "ip"}, orig_sha512_hash: %{type: "keyword"},
mime_type: %{type: "keyword"}, original_format: %{type: "keyword"},
orig_sha512_hash: %{type: "keyword"}, score: %{type: "integer"},
original_format: %{type: "keyword"}, sha512_hash: %{type: "keyword"},
score: %{type: "integer"}, source_url: %{type: "keyword"},
sha512_hash: %{type: "keyword"}, tag_count: %{type: "integer"},
source_url: %{type: "keyword"}, tag_ids: %{type: "keyword"},
tag_count: %{type: "integer"}, tags: %{type: "text", analyzer: "keyword"},
tag_ids: %{type: "keyword"}, true_uploader: %{type: "keyword"},
tags: %{type: "text", analyzer: "keyword"}, true_uploader_id: %{type: "keyword"},
true_uploader: %{type: "keyword"}, updated_at: %{type: "date"},
true_uploader_id: %{type: "keyword"}, uploader: %{type: "keyword"},
updated_at: %{type: "date"}, uploader_id: %{type: "keyword"},
uploader: %{type: "keyword"}, upvoter_ids: %{type: "keyword"},
uploader_id: %{type: "keyword"}, upvoters: %{type: "keyword"},
upvoter_ids: %{type: "keyword"}, upvotes: %{type: "integer"},
upvoters: %{type: "keyword"}, user_id: %{type: "keyword"},
upvotes: %{type: "integer"}, width: %{type: "integer"},
user_id: %{type: "keyword"}, wilson_score: %{type: "float"},
width: %{type: "integer"}, galleries: %{
wilson_score: %{type: "float"}, type: "nested",
galleries: %{ properties: %{
type: "nested", id: %{type: "integer"},
properties: %{ position: %{type: "integer"}
id: %{type: "integer"}, }
position: %{type: "integer"} },
} namespaced_tags: %{
}, properties: %{
namespaced_tags: %{ name: %{type: "keyword"},
properties: %{ name_in_namespace: %{type: "keyword"},
name: %{type: "keyword"}, namespace: %{type: "keyword"}
name_in_namespace: %{type: "keyword"},
namespace: %{type: "keyword"}
}
} }
} }
} }

48
lib/philomena/posts/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Posts.ElasticsearchIndex do
"posts" "posts"
end end
@impl true
def doc_type do
"post"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -21,29 +16,26 @@ defmodule Philomena.Posts.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
post: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, id: %{type: "integer"},
properties: %{ body: %{type: "text", analyzer: "snowball"},
id: %{type: "integer"}, ip: %{type: "ip"},
body: %{type: "text", analyzer: "snowball"}, user_agent: %{type: "keyword"},
ip: %{type: "ip"}, referrer: %{type: "keyword"},
user_agent: %{type: "keyword"}, fingerprint: %{type: "keyword"},
referrer: %{type: "keyword"}, subject: %{type: "text", analyzer: "snowball"},
fingerprint: %{type: "keyword"}, author: %{type: "keyword"},
subject: %{type: "text", analyzer: "snowball"}, topic_position: %{type: "integer"},
author: %{type: "keyword"}, forum_id: %{type: "keyword"},
topic_position: %{type: "integer"}, topic_id: %{type: "keyword"},
forum_id: %{type: "keyword"}, user_id: %{type: "keyword"},
topic_id: %{type: "keyword"}, anonymous: %{type: "boolean"},
user_id: %{type: "keyword"}, updated_at: %{type: "date"},
anonymous: %{type: "boolean"}, created_at: %{type: "date"},
updated_at: %{type: "date"}, deleted: %{type: "boolean"},
created_at: %{type: "date"}, access_level: %{type: "keyword"},
deleted: %{type: "boolean"}, destroyed_content: %{type: "boolean"}
access_level: %{type: "keyword"},
destroyed_content: %{type: "boolean"}
}
} }
} }
} }

40
lib/philomena/reports/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Reports.ElasticsearchIndex do
"reports" "reports"
end end
@impl true
def doc_type do
"report"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -21,25 +16,22 @@ defmodule Philomena.Reports.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
report: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, id: %{type: "integer"},
properties: %{ image_id: %{type: "integer"},
id: %{type: "integer"}, created_at: %{type: "date"},
image_id: %{type: "integer"}, ip: %{type: "ip"},
created_at: %{type: "date"}, fingerprint: %{type: "keyword"},
ip: %{type: "ip"}, state: %{type: "keyword"},
fingerprint: %{type: "keyword"}, user: %{type: "keyword"},
state: %{type: "keyword"}, user_id: %{type: "keyword"},
user: %{type: "keyword"}, admin: %{type: "keyword"},
user_id: %{type: "keyword"}, admin_id: %{type: "keyword"},
admin: %{type: "keyword"}, reportable_type: %{type: "keyword"},
admin_id: %{type: "keyword"}, reportable_id: %{type: "keyword"},
reportable_type: %{type: "keyword"}, open: %{type: "boolean"},
reportable_id: %{type: "keyword"}, reason: %{type: "text", analyzer: "snowball"}
open: %{type: "boolean"},
reason: %{type: "text", analyzer: "snowball"}
}
} }
} }
} }

56
lib/philomena/tags/elasticsearch_index.ex
View file

@ -6,11 +6,6 @@ defmodule Philomena.Tags.ElasticsearchIndex do
"tags" "tags"
end end
@impl true
def doc_type do
"tag"
end
@impl true @impl true
def mapping do def mapping do
%{ %{
@ -29,33 +24,30 @@ defmodule Philomena.Tags.ElasticsearchIndex do
} }
}, },
mappings: %{ mappings: %{
tag: %{ dynamic: false,
_all: %{enabled: false}, properties: %{
dynamic: false, id: %{type: "integer"},
properties: %{ images: %{type: "integer"},
id: %{type: "integer"}, slug: %{type: "keyword"},
images: %{type: "integer"}, name: %{type: "keyword"},
slug: %{type: "keyword"}, name_in_namespace: %{type: "keyword"},
name: %{type: "keyword"}, namespace: %{type: "keyword"},
name_in_namespace: %{type: "keyword"}, aliased_tag: %{type: "keyword"},
namespace: %{type: "keyword"}, aliases: %{type: "keyword"},
aliased_tag: %{type: "keyword"}, implied_tags: %{type: "keyword"},
aliases: %{type: "keyword"}, implied_tag_ids: %{type: "keyword"},
implied_tags: %{type: "keyword"}, implied_by_tags: %{type: "keyword"},
implied_tag_ids: %{type: "keyword"}, category: %{type: "keyword"},
implied_by_tags: %{type: "keyword"}, aliased: %{type: "boolean"},
category: %{type: "keyword"}, analyzed_name: %{
aliased: %{type: "boolean"}, type: "text",
analyzed_name: %{ fields: %{
type: "text", nlp: %{type: "text", analyzer: "tag_snowball"},
fields: %{ ngram: %{type: "keyword"}
nlp: %{type: "text", analyzer: "tag_snowball"}, }
ngram: %{type: "keyword"} },
} description: %{type: "text", analyzer: "snowball"},
}, short_description: %{type: "text", analyzer: "snowball"}
description: %{type: "text", analyzer: "snowball"},
short_description: %{type: "text", analyzer: "snowball"}
}
} }
} }
} }

2
lib/philomena_web/templates/stat/index.html.slime
View file

@ -23,7 +23,7 @@ elixir:
p p
' There are ' There are
span.stat> span.stat>
= number_with_delimiter(@comment_aggs["hits"]["total"]) = number_with_delimiter(@comment_aggs["hits"]["total"]["value"])
' comments on the site. Of these, ' comments on the site. Of these,
=> number_with_delimiter(cmt_bucket["deleted"]["doc_count"]) => number_with_delimiter(cmt_bucket["deleted"]["doc_count"])
' have been deleted. ' have been deleted.