From 136bfa7eda55f78d4fefc49f505e34e088f43efe Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 27 Mar 2023 19:14:41 -0400 Subject: [PATCH] Fix id generation in fast indexer --- index/comments.mk | 2 +- index/galleries.mk | 2 +- index/images.mk | 2 +- index/posts.mk | 2 +- index/reports.mk | 2 +- index/tags.mk | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/index/comments.mk b/index/comments.mk index 0d55d20d..5699ea35 100644 --- a/index/comments.mk +++ b/index/comments.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=comments.jsonl --output=http://localhost:9200/ --output-index=comments --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=comments.jsonl --output=http://localhost:9200/ --output-index=comments --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata authors tags psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_comments.jsonb_object_agg(object) from temp_comments.comment_search_json group by comment_id) to stdout;' > comments.jsonl diff --git a/index/galleries.mk b/index/galleries.mk index b6155080..1447f1a9 100644 --- a/index/galleries.mk +++ b/index/galleries.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=galleries.jsonl --output=http://localhost:9200/ --output-index=galleries --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=galleries.jsonl --output=http://localhost:9200/ --output-index=galleries --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata subscribers images psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_galleries.jsonb_object_agg(object) from temp_galleries.gallery_search_json group by gallery_id) to stdout;' > galleries.jsonl diff --git a/index/images.mk b/index/images.mk index 35466379..c1f27a49 100644 --- a/index/images.mk +++ b/index/images.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=images.jsonl --output=http://localhost:9200/ --output-index=images --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=images.jsonl --output=http://localhost:9200/ --output-index=images --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata true_uploaders uploaders deleters galleries tags hides upvotes downvotes faves tag_names psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_images.jsonb_object_agg(object) from temp_images.image_search_json group by image_id) to stdout;' > images.jsonl diff --git a/index/posts.mk b/index/posts.mk index fda8e8ee..939066cb 100644 --- a/index/posts.mk +++ b/index/posts.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=posts.jsonl --output=http://localhost:9200/ --output-index=posts --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=posts.jsonl --output=http://localhost:9200/ --output-index=posts --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata authors psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_posts.jsonb_object_agg(object) from temp_posts.post_search_json group by post_id) to stdout;' > posts.jsonl diff --git a/index/reports.mk b/index/reports.mk index 139f194a..d8d810da 100644 --- a/index/reports.mk +++ b/index/reports.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=reports.jsonl --output=http://localhost:9200/ --output-index=reports --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=reports.jsonl --output=http://localhost:9200/ --output-index=reports --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata image_ids comment_image_ids psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_reports.jsonb_object_agg(object) from temp_reports.report_search_json group by report_id) to stdout;' > reports.jsonl diff --git a/index/tags.mk b/index/tags.mk index 0c5413f9..1b184310 100644 --- a/index/tags.mk +++ b/index/tags.mk @@ -5,7 +5,7 @@ ELASTICDUMP ?= elasticdump all: import_es import_es: dump_jsonl - $(ELASTICDUMP) --input=tags.jsonl --output=http://localhost:9200/ --output-index=tags --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc)" + $(ELASTICDUMP) --input=tags.jsonl --output=http://localhost:9200/ --output-index=tags --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id" dump_jsonl: metadata aliases implied_tags implied_by_tags psql $(DATABASE) -v ON_ERROR_STOP=1 <<< 'copy (select temp_tags.jsonb_object_agg(object) from temp_tags.tag_search_json group by tag_id) to stdout;' > tags.jsonl