From 3ee8179cc8d0b67e752cf11b2162bfeed461026a Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 12 Feb 2022 13:44:42 -0500
Subject: [PATCH] Add uploader task

---
 docker/app/run-development          |   2 +-
 lib/mix/tasks/upload_to_s3.ex       | 115 ++++++++++++++++++++++++++++
 lib/philomena/images/thumbnailer.ex |   6 +-
 3 files changed, 119 insertions(+), 4 deletions(-)
 create mode 100644 lib/mix/tasks/upload_to_s3.ex

diff --git a/docker/app/run-development b/docker/app/run-development
index bc209c9a..9486d334 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -43,7 +43,7 @@ until wget -qO - elasticsearch:9200; do
   sleep 2
 done
 
-until wget -qO - files/philomena; do
+until wget -qO /dev/null files/philomena; do
   echo -n "."
   sleep 2
 done
diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
new file mode 100644
index 00000000..166be286
--- /dev/null
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -0,0 +1,115 @@
+defmodule Mix.Tasks.UploadToS3 do
+  use Mix.Task
+
+  alias Philomena.{
+    Adverts.Advert,
+    Badges.Badge,
+    Images.Image,
+    Tags.Tag,
+    Users.User
+  }
+
+  alias Philomena.Images.Thumbnailer
+  alias Philomena.Mime
+  alias Philomena.Batch
+  alias ExAws.S3
+  import Ecto.Query
+
+  @shortdoc "Dumps existing image files to S3 storage backend"
+  @requirements ["app.start"]
+  @impl Mix.Task
+  def run([concurrency | args]) do
+    {concurrency, _} = Integer.parse(concurrency)
+
+    if Enum.member?(args, "--adverts") do
+      file_root = Application.fetch_env!(:philomena, :advert_file_root)
+      new_file_root = System.get_env("NEW_ADVERT_FILE_ROOT", "adverts")
+
+      IO.puts "\nAdverts:"
+      upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--avatars") do
+      file_root = Application.fetch_env!(:philomena, :avatar_file_root)
+      new_file_root = System.get_env("NEW_AVATAR_FILE_ROOT", "avatars")
+
+      IO.puts "\nAvatars:"
+      upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, "avatar")
+    end
+
+    if Enum.member?(args, "--badges") do
+      file_root = Application.fetch_env!(:philomena, :badge_file_root)
+      new_file_root = System.get_env("NEW_BADGE_FILE_ROOT", "badges")
+
+      IO.puts "\nBadges:"
+      upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--tags") do
+      file_root = Application.fetch_env!(:philomena, :tag_file_root)
+      new_file_root = System.get_env("NEW_TAG_FILE_ROOT", "tags")
+
+      IO.puts "\nTags:"
+      upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--images") do
+      # Temporarily adjust the file root so that the thumbs are picked up
+      file_root = Application.fetch_env!(:philomena, :image_file_root) <> "thumbs"
+      Application.put_env(:philomena, :image_file_root, file_root)
+
+      new_file_root = System.get_env("NEW_IMAGE_FILE_ROOT", "images")
+
+      IO.puts "\nImages:"
+      upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
+    end
+  end
+
+  defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
+    Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+      Task.async_stream(models, &upload_typical_model(&1, file_root, new_file_root, field_name))
+
+      IO.write "\r#{hd(models).id}"
+    end)
+  end
+
+  defp upload_typical_model(model, file_root, new_file_root, field_name) do
+    path = Path.join(file_root, Map.fetch!(model, field_name))
+
+    if File.exists?(path) do
+      put_file(path, Path.join(new_file_root, field_name))
+    end
+  end
+
+  defp upload_images(queryable, batch_size, file_root, new_file_root) do
+    Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+      Task.async_stream(models, &upload_image_model(&1, file_root, new_file_root))
+
+      IO.write "\r#{hd(models).id}"
+    end)
+  end
+
+  defp upload_image_model(model, file_root, new_file_root) do
+    path_prefix = Thumbnailer.image_thumb_prefix(model)
+
+    Thumbnailer.all_versions(model)
+    |> Enum.map(fn version ->
+      path = Path.join([file_root, path_prefix, version])
+      new_path = Path.join([new_file_root, path_prefix, version])
+
+      put_file(path, new_path)
+    end)
+  end
+
+  defp put_file(path, uploaded_path) do
+    mime = Mime.file(path)
+    contents = File.read!(path)
+
+    S3.put_object(bucket(), uploaded_path, contents, acl: :public_read, content_type: mime)
+    |> ExAws.request!()
+  end
+
+  defp bucket do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
+end
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 9d80a070..a093612d 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -153,7 +153,7 @@ defmodule Philomena.Images.Thumbnailer do
     end)
   end
 
-  defp all_versions(image) do
+  def all_versions(image) do
     generated = Processors.versions(image.image_mime_type, generated_sizes(image))
     full = ["full.#{image.image_format}"]
 
@@ -163,10 +163,10 @@ defmodule Philomena.Images.Thumbnailer do
   # This method wraps the following two for code that doesn't care
   # and just wants the files (most code should take this path)
 
-  defp image_thumb_prefix(%{hidden_from_users: true} = image),
+  def image_thumb_prefix(%{hidden_from_users: true} = image),
     do: hidden_image_thumb_prefix(image, image.hidden_image_key)
 
-  defp image_thumb_prefix(image),
+  def image_thumb_prefix(image),
     do: visible_image_thumb_prefix(image)
 
   # These methods handle the actual distinction between the two