philomena/index/reports.mk

DATABASE ?= philomena
OPENSEARCH_URL ?= http://localhost:9200/
ELASTICDUMP ?= elasticdump
.ONESHELL:

all: import_es

import_es: dump_jsonl
	$(ELASTICDUMP) --input=reports.jsonl --output=$OPENSEARCH_URL --output-index=reports --limit 10000 --retryAttempts=5 --type=data --transform="doc._source = Object.assign({},doc); doc._id = doc.id"

dump_jsonl: metadata image_ids comment_image_ids
	psql $(DATABASE) -v ON_ERROR_STOP=1 -c 'copy (select temp_reports.jsonb_object_agg(object) from temp_reports.report_search_json group by report_id) to stdout;' > reports.jsonl
	psql $(DATABASE) -v ON_ERROR_STOP=1 -c 'drop schema temp_reports cascade;'
	sed -i reports.jsonl -e 's/\\\\/\\/g'

metadata: report_search_json
	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
		insert into temp_reports.report_search_json (report_id, object) select r.id, jsonb_build_object(
			'id', r.id,
			'created_at', r.created_at,
			'ip', r.ip,
			'state', r.state,
			'user', lower(u.name),
			'user_id', r.user_id,
			'admin', lower(a.name),
			'admin_id', r.admin_id,
			'reportable_type', r.reportable_type,
			'reportable_id', r.reportable_id,
			'fingerprint', r.fingerprint,
			'open', r.open,
			'reason', r.reason
		) from reports r left join users u on r.user_id=u.id left join users a on r.admin_id=a.id;
	SQL

image_ids: report_search_json
	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
		insert into temp_reports.report_search_json (report_id, object) select r.id, jsonb_build_object('image_id', r.reportable_id) from reports r where r.reportable_type = 'Image';
	SQL

comment_image_ids: report_search_json
	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
		insert into temp_reports.report_search_json (report_id, object) select r.id, jsonb_build_object('image_id', c.image_id) from reports r inner join comments c on c.id = r.reportable_id where r.reportable_type = 'Comment';
	SQL

report_search_json:
	psql $(DATABASE) -v ON_ERROR_STOP=1 <<-SQL
		drop schema if exists temp_reports cascade;
		create schema temp_reports;
		create unlogged table temp_reports.report_search_json (report_id bigint not null, object jsonb not null);
		create or replace aggregate temp_reports.jsonb_object_agg(jsonb) (sfunc = 'jsonb_concat', stype = jsonb, initcond='{}');
	SQL