PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2025-01-20 06:37:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Use concurrency at the lowest level

Browse source
This commit is contained in:
Liam 2025-01-11 12:14:26 -05:00
parent d9309a275e
commit 4cce622332
1 changed files with 21 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
lib/philomena/search_indexer.ex
View file

@ -63,11 +63,7 @@ defmodule Philomena.SearchIndexer do
@spec recreate_reindex_all_destructive! :: :ok @spec recreate_reindex_all_destructive! :: :ok
def recreate_reindex_all_destructive! do def recreate_reindex_all_destructive! do
@schemas @schemas
|> Task.async_stream( |> Stream.map(&recreate_reindex_schema_destructive!/1)
&recreate_reindex_schema_destructive!/1,
ordered: false,
timeout: :infinity
)
|> Stream.run() |> Stream.run()
end end
@ -101,11 +97,7 @@ defmodule Philomena.SearchIndexer do
@spec reindex_all :: :ok @spec reindex_all :: :ok
def reindex_all do def reindex_all do
@schemas @schemas
|> Task.async_stream( |> Stream.map(&reindex_schema/1)
&reindex_schema/1,
ordered: false,
timeout: :infinity
)
|> Stream.run() |> Stream.run()
end end
@ -126,11 +118,16 @@ defmodule Philomena.SearchIndexer do
Report Report
|> preload([:user, :admin]) |> preload([:user, :admin])
|> Batch.record_batches(batch_size: @batch_sizes[Report]) |> Batch.record_batches(batch_size: @batch_sizes[Report])
|> Enum.each(fn records -> |> Task.async_stream(
records fn records ->
|> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type]) records
|> Enum.map(&Search.index_document(&1, Report)) |> Polymorphic.load_polymorphic(reportable: [reportable_id: :reportable_type])
end) |> Enum.map(&Search.index_document(&1, Report))
end,
timeout: :infinity,
max_concurrency: max_concurrency()
)
|> Stream.run()
end end
def reindex_schema(schema) when schema in @schemas do def reindex_schema(schema) when schema in @schemas do
@ -139,6 +136,14 @@ defmodule Philomena.SearchIndexer do
schema schema
|> preload(^context.indexing_preloads()) |> preload(^context.indexing_preloads())
|> Search.reindex(schema, batch_size: @batch_sizes[schema]) |> Search.reindex(schema,
batch_size: @batch_sizes[schema],
max_concurrency: max_concurrency()
)
end
@spec max_concurrency() :: pos_integer()
defp max_concurrency do
System.schedulers_online()
end end
end end