From f80d8577a837fc14df3928fc816e941579ad553e Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Tue, 8 Feb 2022 19:08:53 -0500 Subject: [PATCH] Finish cleanup of thumbnailer --- lib/mix/tasks/recalculate_intensities.ex | 53 ------------------------ lib/philomena/filename.ex | 16 +------ lib/philomena/images.ex | 11 ++--- lib/philomena/images/thumbnailer.ex | 18 +++++++- lib/philomena/images/uploader.ex | 9 ++-- 5 files changed, 27 insertions(+), 80 deletions(-) delete mode 100644 lib/mix/tasks/recalculate_intensities.ex diff --git a/lib/mix/tasks/recalculate_intensities.ex b/lib/mix/tasks/recalculate_intensities.ex deleted file mode 100644 index ad3ee171..00000000 --- a/lib/mix/tasks/recalculate_intensities.ex +++ /dev/null @@ -1,53 +0,0 @@ -defmodule Mix.Tasks.RecalculateIntensities do - use Mix.Task - - alias Philomena.Images.{Image, Thumbnailer} - alias Philomena.ImageIntensities.ImageIntensity - alias Philomena.Batch - alias Philomena.Repo - - import Ecto.Query - - @shortdoc "Recalculates all intensities for reverse search." - @requirements ["app.start"] - @impl Mix.Task - def run(_args) do - Batch.record_batches(Image, fn batch -> - batch - |> Stream.with_index() - |> Stream.each(fn {image, i} -> - image_file = - cond do - image.image_mime_type in ["image/png", "image/jpeg"] -> - Thumbnailer.image_file(image) - - true -> - Path.join(Thumbnailer.image_thumb_dir(image), "rendered.png") - end - - case System.cmd("image-intensities", [image_file]) do - {output, 0} -> - [nw, ne, sw, se] = - output - |> String.trim() - |> String.split("\t") - |> Enum.map(&String.to_float/1) - - ImageIntensity - |> where(image_id: ^image.id) - |> Repo.update_all(set: [nw: nw, ne: ne, sw: sw, se: se]) - - _ -> - :err - end - - if rem(i, 100) == 0 do - IO.write("\r#{image.id}") - end - end) - |> Stream.run() - end) - - IO.puts("\nDone") - end -end diff --git a/lib/philomena/filename.ex b/lib/philomena/filename.ex index 43b081a1..ea0d230d 100644 --- a/lib/philomena/filename.ex +++ b/lib/philomena/filename.ex @@ -8,8 +8,7 @@ defmodule Philomena.Filename do [ time_identifier(DateTime.utc_now()), "/", - usec_identifier(), - pid_identifier(), + UUID.uuid1(), ".", extension ] @@ -19,17 +18,4 @@ defmodule Philomena.Filename do defp time_identifier(time) do Enum.join([time.year, time.month, time.day], "/") end - - defp usec_identifier do - DateTime.utc_now() - |> DateTime.to_unix(:microsecond) - |> to_string() - end - - defp pid_identifier do - self() - |> :erlang.pid_to_list() - |> to_string() - |> String.replace(~r/[^0-9]/, "") - end end diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex index 9feda6ab..a244f280 100644 --- a/lib/philomena/images.ex +++ b/lib/philomena/images.ex @@ -13,7 +13,6 @@ defmodule Philomena.Images do alias Philomena.ImagePurgeWorker alias Philomena.DuplicateReports.DuplicateReport alias Philomena.Images.Image - alias Philomena.Images.Hider alias Philomena.Images.Uploader alias Philomena.Images.Tagging alias Philomena.Images.Thumbnailer @@ -198,7 +197,7 @@ defmodule Philomena.Images do {:ok, image} -> Uploader.unpersist_old_upload(image) purge_files(image, image.hidden_image_key) - Hider.destroy_thumbnails(image) + Thumbnailer.destroy_thumbnails(image) {:ok, image} @@ -539,7 +538,7 @@ defmodule Philomena.Images do defp process_after_hide(result) do case result do {:ok, %{image: image, tags: tags, reports: {_count, reports}} = result} -> - Hider.hide_thumbnails(image, image.hidden_image_key) + Thumbnailer.hide_thumbnails(image, image.hidden_image_key) Comments.reindex_comments(image) Reports.reindex_reports(reports) @@ -590,7 +589,7 @@ defmodule Philomena.Images do |> Repo.transaction() |> case do {:ok, %{image: image, tags: tags}} -> - Hider.unhide_thumbnails(image, key) + Thumbnailer.unhide_thumbnails(image, key) reindex_image(image) purge_files(image, image.hidden_image_key) @@ -774,7 +773,9 @@ defmodule Philomena.Images do end def perform_purge(files) do - Hider.purge_cache(files) + {_out, 0} = System.cmd("purge-cache", [Jason.encode!(%{files: files})]) + + :ok end alias Philomena.Images.Subscription diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex index 2a14a4cb..f078ddf1 100644 --- a/lib/philomena/images/thumbnailer.ex +++ b/lib/philomena/images/thumbnailer.ex @@ -36,6 +36,13 @@ defmodule Philomena.Images.Thumbnailer do end) end + def thumbnail_urls(image, hidden_key) do + Processors.versions(image.image_mime_type, generated_sizes(image)) + |> Enum.map(fn name -> + Path.join(image_url_base(image, hidden_key), name) + end) + end + def hide_thumbnails(image, key) do moved_files = Processors.versions(image.image_mime_type, generated_sizes(image)) @@ -120,7 +127,7 @@ defmodule Philomena.Images.Thumbnailer do tempfile end - defp upload_file(image, file, version_name) do + def upload_file(image, file, version_name) do path = Path.join(image_thumb_prefix(image), version_name) file @@ -164,12 +171,21 @@ defmodule Philomena.Images.Thumbnailer do defp visible_image_thumb_prefix(%Image{created_at: created_at, id: id}), do: Path.join([image_file_root(), time_identifier(created_at), to_string(id)]) + defp image_url_base(%Image{created_at: created_at, id: id}, nil), + do: Path.join([image_url_root(), time_identifier(created_at), to_string(id)]) + + defp image_url_base(%Image{created_at: created_at, id: id}, key), + do: Path.join([image_url_root(), time_identifier(created_at), "#{id}-#{key}"]) + defp time_identifier(time), do: Enum.join([time.year, time.month, time.day], "/") defp image_file_root, do: Application.fetch_env!(:philomena, :image_file_root) + defp image_url_root, + do: Application.fetch_env!(:philomena, :image_url_root) + defp bucket, do: Application.fetch_env!(:philomena, :s3_bucket) end diff --git a/lib/philomena/images/uploader.ex b/lib/philomena/images/uploader.ex index 35b96394..8419b805 100644 --- a/lib/philomena/images/uploader.ex +++ b/lib/philomena/images/uploader.ex @@ -3,6 +3,7 @@ defmodule Philomena.Images.Uploader do Upload and processing callback logic for Images. """ + alias Philomena.Images.Thumbnailer alias Philomena.Images.Image alias Philomena.Uploader @@ -11,14 +12,10 @@ defmodule Philomena.Images.Uploader do end def persist_upload(image) do - Uploader.persist_upload(image, image_file_root(), "image") + Thumbnailer.upload_file(image, image.uploaded_image, "full.#{image.image_format}") end def unpersist_old_upload(image) do - Uploader.unpersist_old_upload(image, image_file_root(), "image") - end - - defp image_file_root do - Application.get_env(:philomena, :image_file_root) + Thumbnailer.destroy_thumbnails(image) end end