From d961d6151f7b236b06ddc10ecd091fa32074e97d Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 6 Feb 2022 13:27:01 -0500
Subject: [PATCH 01/39] Docker setup

---
 .gitignore                           |  1 +
 config/runtime.exs                   |  6 ++++++
 docker-compose.yml                   | 13 +++++++++++++
 docker/app/run-development           |  8 ++++++++
 docker/files/Dockerfile              |  2 ++
 docker/files/run-docker-container.sh | 28 ++++++++++++++++++++++++++++
 mix.exs                              |  3 +++
 mix.lock                             |  3 +++
 8 files changed, 64 insertions(+)
 create mode 100644 docker/files/Dockerfile
 create mode 100644 docker/files/run-docker-container.sh

diff --git a/.gitignore b/.gitignore
index 6b5b12c6..354e8069 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ npm-debug.log
 # we ignore priv/static. You may want to comment
 # this depending on your deployment strategy.
 /priv/static/
+/priv/s3
 
 # Intellij IDEA
 .idea
diff --git a/config/runtime.exs b/config/runtime.exs
index d8719a8e..bece7870 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -67,6 +67,12 @@ if is_nil(System.get_env("START_WORKER")) do
   config :exq, queues: []
 end
 
+# S3 config
+config :ex_aws, :s3,
+  scheme: System.fetch_env!("S3_SCHEME"),
+  host: System.fetch_env!("S3_HOST"),
+  port: System.fetch_env!("S3_PORT")
+
 if config_env() != :test do
   # Database config
   config :philomena, Philomena.Repo,
diff --git a/docker-compose.yml b/docker-compose.yml
index d71b0c03..84c6d7c9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,6 +35,12 @@ services:
       - MAILER_ADDRESS=noreply@philomena.local
       - START_ENDPOINT=true
       - SITE_DOMAINS=localhost
+      - S3_SCHEME=http
+      - S3_HOST=files
+      - S3_PORT=80
+      - S3_BUCKET=philomena
+      - AWS_ACCESS_KEY_ID=local-identity
+      - AWS_SECRET_ACCESS_KEY=local-credential
     working_dir: /srv/philomena
     tty: true
     volumes:
@@ -71,6 +77,13 @@ services:
     logging:
       driver: "none"
 
+  files:
+    build:
+      context: .
+      dockerfile: ./docker/app/Dockerfile
+    volumes:
+      - .:/srv/philomena
+
   web:
     build:
       context: .
diff --git a/docker/app/run-development b/docker/app/run-development
index dc3e7009..5e171745 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -1,5 +1,13 @@
 #!/usr/bin/env sh
 
+# Create S3 dirs
+mkdir -p /srv/philomena/priv/s3/philomena
+ln -s /srv/philomena/priv/system/images/thumbs /srv/philomena/priv/s3/philomena/images
+ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/adverts
+ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/avatars
+ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/badges
+ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/tags
+
 # For compatibility with musl libc
 export CARGO_FEATURE_DISABLE_INITIAL_EXEC_TLS=1
 export CARGO_HOME=/srv/philomena/.cargo
diff --git a/docker/files/Dockerfile b/docker/files/Dockerfile
new file mode 100644
index 00000000..9316f2c1
--- /dev/null
+++ b/docker/files/Dockerfile
@@ -0,0 +1,2 @@
+FROM andrewgaul/s3proxy:sha-2e61c38
+COPY docker/files/run-docker-container.sh /opt/s3proxy/run-docker-container.sh
diff --git a/docker/files/run-docker-container.sh b/docker/files/run-docker-container.sh
new file mode 100644
index 00000000..171cbd55
--- /dev/null
+++ b/docker/files/run-docker-container.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+# forked because it doesn't let me configure the filesystem basedir via env
+mkdir -p /srv/philomena/priv/s3
+
+exec java \
+    -DLOG_LEVEL="${LOG_LEVEL}" \
+    -Ds3proxy.endpoint="${S3PROXY_ENDPOINT}" \
+    -Ds3proxy.virtual-host="${S3PROXY_VIRTUALHOST}" \
+    -Ds3proxy.authorization="${S3PROXY_AUTHORIZATION}" \
+    -Ds3proxy.identity="${S3PROXY_IDENTITY}" \
+    -Ds3proxy.credential="${S3PROXY_CREDENTIAL}" \
+    -Ds3proxy.cors-allow-all="${S3PROXY_CORS_ALLOW_ALL}" \
+    -Ds3proxy.cors-allow-origins="${S3PROXY_CORS_ALLOW_ORIGINS}" \
+    -Ds3proxy.cors-allow-methods="${S3PROXY_CORS_ALLOW_METHODS}" \
+    -Ds3proxy.cors-allow-headers="${S3PROXY_CORS_ALLOW_HEADERS}" \
+    -Ds3proxy.ignore-unknown-headers="${S3PROXY_IGNORE_UNKNOWN_HEADERS}" \
+    -Djclouds.provider="${JCLOUDS_PROVIDER}" \
+    -Djclouds.identity="${JCLOUDS_IDENTITY}" \
+    -Djclouds.credential="${JCLOUDS_CREDENTIAL}" \
+    -Djclouds.endpoint="${JCLOUDS_ENDPOINT}" \
+    -Djclouds.region="${JCLOUDS_REGION}" \
+    -Djclouds.regions="${JCLOUDS_REGIONS}" \
+    -Djclouds.keystone.version="${JCLOUDS_KEYSTONE_VERSION}" \
+    -Djclouds.keystone.scope="${JCLOUDS_KEYSTONE_SCOPE}" \
+    -Djclouds.keystone.project-domain-name="${JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME}" \
+    -Djclouds.filesystem.basedir="/srv/philomena/priv/s3" \
+    -jar /opt/s3proxy/s3proxy \
+    --properties /dev/null
diff --git a/mix.exs b/mix.exs
index fb40561f..aa3ce11e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -69,6 +69,9 @@ defmodule Philomena.MixProject do
       {:castore, "~> 0.1"},
       {:mint, "~> 1.2"},
       {:exq, "~> 0.14"},
+      {:ex_aws, "~> 2.0"},
+      {:ex_aws_s3, "~> 2.0"},
+      {:sweet_xml, "~> 0.7"},
 
       # Markdown
       {:rustler, "~> 0.22"},
diff --git a/mix.lock b/mix.lock
index 0b5359b3..8e0f530e 100644
--- a/mix.lock
+++ b/mix.lock
@@ -27,6 +27,8 @@
   "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
   "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
+  "ex_aws": {:hex, :ex_aws, "2.2.10", "064139724335b00b6665af7277189afc9ed507791b1ccf2698dadc7c8ad892e8", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "98acb63f74b2f0822be219c5c2f0e8d243c2390f5325ad0557b014d3360da47e"},
+  "ex_aws_s3": {:hex, :ex_aws_s3, "2.3.2", "92a63b72d763b488510626d528775b26831f5c82b066a63a3128054b7a09de28", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "b235b27131409bcc293c343bf39f1fbdd32892aa237b3f13752e914dc2979960"},
   "exq": {:hex, :exq, "0.16.1", "140d78e95a538d265d23a1e7a090f501c40c799f269b8b503f4cbd962447e708", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 5.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "ce70231e2892130b0f80d1bbc8d6ddd22d89d1157b32e783a933eaadb31bc821"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
   "gen_smtp": {:hex, :gen_smtp, "1.1.1", "bf9303c31735100631b1d708d629e4c65944319d1143b5c9952054f4a1311d85", [:rebar3], [{:hut, "1.3.0", [hex: :hut, repo: "hexpm", optional: false]}, {:ranch, ">= 1.7.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "51bc50cc017efd4a4248cbc39ea30fb60efa7d4a49688986fafad84434ff9ab7"},
@@ -73,6 +75,7 @@
   "slime": {:hex, :slime, "1.3.0", "153cebb4a837efaf55fb09dff0d79374ad74af835a0288feccbfd9cf606446f9", [:mix], [{:neotoma, "~> 1.7", [hex: :neotoma, repo: "hexpm", optional: false]}], "hexpm", "303b58f05d740a5fe45165bcadfe01da174f1d294069d09ebd7374cd36990a27"},
   "sobelow": {:hex, :sobelow, "0.11.1", "23438964486f8112b41e743bbfd402da3e5b296fdc9eacab29914b79c48916dd", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9897363a7eff96f4809304a90aad819e2ad5e5d24db547af502885146746a53c"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
+  "sweet_xml": {:hex, :sweet_xml, "0.7.2", "4729f997286811fabdd8288f8474e0840a76573051062f066c4b597e76f14f9f", [:mix], [], "hexpm", "6894e68a120f454534d99045ea3325f7740ea71260bc315f82e29731d570a6e8"},
   "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"},
   "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"},
   "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm", "f1e3dabef71fb510d015fad18c0e05e7c57281001141504c6b69d94e99750a07"},

From 6e4771a57aefc0106cee0e4975230d4135586e8f Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 7 Feb 2022 22:12:40 -0500
Subject: [PATCH 02/39] Allow processors to indicate the thumbnail types they
 can generate

---
 config/runtime.exs                  |  2 +-
 docker-compose.yml                  | 15 +++++------
 lib/philomena/images/thumbnailer.ex | 42 +++++++++++++++++++++--------
 lib/philomena/processors.ex         |  9 +++++++
 lib/philomena/processors/gif.ex     |  6 +++++
 lib/philomena/processors/jpeg.ex    |  4 +++
 lib/philomena/processors/png.ex     |  4 +++
 lib/philomena/processors/svg.ex     |  6 +++++
 lib/philomena/processors/webm.ex    | 11 ++++++++
 lib/philomena/uploader.ex           | 23 +++++++++-------
 10 files changed, 93 insertions(+), 29 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index bece7870..9605d362 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -17,7 +17,6 @@ config :philomena,
   elasticsearch_url: System.get_env("ELASTICSEARCH_URL", "http://localhost:9200"),
   advert_file_root: System.fetch_env!("ADVERT_FILE_ROOT"),
   avatar_file_root: System.fetch_env!("AVATAR_FILE_ROOT"),
-  channel_url_root: System.fetch_env!("CHANNEL_URL_ROOT"),
   badge_file_root: System.fetch_env!("BADGE_FILE_ROOT"),
   password_pepper: System.fetch_env!("PASSWORD_PEPPER"),
   avatar_url_root: System.fetch_env!("AVATAR_URL_ROOT"),
@@ -32,6 +31,7 @@ config :philomena,
   tag_url_root: System.fetch_env!("TAG_URL_ROOT"),
   redis_host: System.get_env("REDIS_HOST", "localhost"),
   proxy_host: System.get_env("PROXY_HOST"),
+  s3_bucket: System.fetch_env!("S3_BUCKET")
   camo_host: System.get_env("CAMO_HOST"),
   camo_key: System.get_env("CAMO_KEY"),
   cdn_host: System.fetch_env!("CDN_HOST")
diff --git a/docker-compose.yml b/docker-compose.yml
index 84c6d7c9..9a27584b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,17 +17,16 @@ services:
       - PASSWORD_PEPPER=dn2e0EpZrvBLoxUM3gfQveBhjf0bG/6/bYhrOyq3L3hV9hdo/bimJ+irbDWsuXLP
       - TUMBLR_API_KEY=fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4
       - OTP_SECRET_KEY=Wn7O/8DD+qxL0X4X7bvT90wOkVGcA90bIHww4twR03Ci//zq7PnMw8ypqyyT/b/C
-      - ADVERT_FILE_ROOT=priv/static/system/images/adverts
-      - AVATAR_FILE_ROOT=priv/static/system/images/avatars
-      - BADGE_FILE_ROOT=priv/static/system/images
-      - IMAGE_FILE_ROOT=priv/static/system/images
-      - TAG_FILE_ROOT=priv/static/system/images
-      - CHANNEL_URL_ROOT=/media
+      - ADVERT_FILE_ROOT=adverts
+      - AVATAR_FILE_ROOT=avatars
+      - BADGE_FILE_ROOT=badges
+      - IMAGE_FILE_ROOT=images
+      - TAG_FILE_ROOT=tags
       - AVATAR_URL_ROOT=/avatars
       - ADVERT_URL_ROOT=/spns
       - IMAGE_URL_ROOT=/img
-      - BADGE_URL_ROOT=/media
-      - TAG_URL_ROOT=/media
+      - BADGE_URL_ROOT=/badges
+      - TAG_URL_ROOT=/tags
       - ELASTICSEARCH_URL=http://elasticsearch:9200
       - REDIS_HOST=redis
       - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 97da4083..c1c8cdeb 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -10,6 +10,7 @@ defmodule Philomena.Images.Thumbnailer do
   alias Philomena.Analyzers
   alias Philomena.Sha512
   alias Philomena.Repo
+  alias ExAws.S3
 
   @versions [
     thumb_tiny: {50, 50},
@@ -18,27 +19,24 @@ defmodule Philomena.Images.Thumbnailer do
     small: {320, 240},
     medium: {800, 600},
     large: {1280, 1024},
-    tall: {1024, 4096},
-    full: nil
+    tall: {1024, 4096}
   ]
 
   def thumbnail_versions do
-    Enum.filter(@versions, fn {_name, dimensions} ->
-      not is_nil(dimensions)
-    end)
+    @versions
   end
 
-  def thumbnail_urls(image, hidden_key) do
-    Path.join([image_thumb_dir(image), "*"])
-    |> Path.wildcard()
-    |> Enum.map(fn version_name ->
-      Path.join([image_url_base(image, hidden_key), Path.basename(version_name)])
+  # A list of version sizes that should be generated for the image,
+  # based on its dimensions. The processor can generate a list of paths.
+  def generated_sizes(%{image_width: image_width, image_height: image_height}) do
+    Enum.filter(@versions, fn
+      {_name, {width, height}} -> image_width > width or image_height > height
     end)
   end
 
   def generate_thumbnails(image_id) do
     image = Repo.get!(Image, image_id)
-    file = image_file(image)
+    file = download_image_file(image)
     {:ok, analysis} = Analyzers.analyze(file)
 
     apply_edit_script(image, Processors.process(analysis, file, @versions))
@@ -135,6 +133,24 @@ defmodule Philomena.Images.Thumbnailer do
   def image_file(%Image{image: image}),
     do: Path.join(image_file_root(), image)
 
+  defp download_image_file(%Image{image: path} = image) do
+    tempfile = Briefly.create!(extname: "." <> image.image_format)
+    path = Path.join(image_file_root(), path)
+
+    ExAws.request!(S3.download_file(bucket(), path, tempfile))
+
+    tempfile
+  end
+
+  defp upload_image_file(%Image{image: path}, new_file) do
+    path = Path.join(image_file_root(), path)
+
+    new_file
+    |> S3.Upload.stream_file()
+    |> S3.upload(bucket(), path, acl: :public_read)
+    |> ExAws.request!()
+  end
+
   def image_thumb_dir(%Image{
         created_at: created_at,
         id: id,
@@ -163,4 +179,8 @@ defmodule Philomena.Images.Thumbnailer do
 
   defp image_url_root,
     do: Application.get_env(:philomena, :image_url_root)
+
+  defp bucket() do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
 end
diff --git a/lib/philomena/processors.ex b/lib/philomena/processors.ex
index 0a6b4afb..531d0533 100644
--- a/lib/philomena/processors.ex
+++ b/lib/philomena/processors.ex
@@ -40,6 +40,15 @@ defmodule Philomena.Processors do
   def processor("video/webm"), do: Webm
   def processor(_content_type), do: nil
 
+  @doc """
+  Takes an analyzer and version list and generates a list of versions to be
+  generated (e.g., ["thumb.png"]). List contents differ based on file type.
+  """
+  @spec versions(map(), keyword) :: [String.t()]
+  def versions(analysis, valid_sizes) do
+    processor(analysis.mime_type).versions(valid_sizes)
+  end
+
   @doc """
   Takes an analyzer, file path, and version list and runs the appropriate
   processor's process/3.
diff --git a/lib/philomena/processors/gif.ex b/lib/philomena/processors/gif.ex
index 7e1f1b36..757d6968 100644
--- a/lib/philomena/processors/gif.ex
+++ b/lib/philomena/processors/gif.ex
@@ -1,6 +1,12 @@
 defmodule Philomena.Processors.Gif do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    sizes
+    |> Enum.map(fn {name, _} -> "#{name}.gif" end)
+    |> Kernel.++(["full.webm", "full.mp4", "rendered.png"])
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     duration = analysis.duration
diff --git a/lib/philomena/processors/jpeg.ex b/lib/philomena/processors/jpeg.ex
index e4dde2ff..18257d9f 100644
--- a/lib/philomena/processors/jpeg.ex
+++ b/lib/philomena/processors/jpeg.ex
@@ -1,6 +1,10 @@
 defmodule Philomena.Processors.Jpeg do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    Enum.map(sizes, fn {name, _} -> "#{name}.jpg" end)
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     stripped = optimize(strip(file))
diff --git a/lib/philomena/processors/png.ex b/lib/philomena/processors/png.ex
index 90e739ae..91d2c87f 100644
--- a/lib/philomena/processors/png.ex
+++ b/lib/philomena/processors/png.ex
@@ -1,6 +1,10 @@
 defmodule Philomena.Processors.Png do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    Enum.map(sizes, fn {name, _} -> "#{name}.png" end)
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     animated? = analysis.animated?
diff --git a/lib/philomena/processors/svg.ex b/lib/philomena/processors/svg.ex
index 0a16d795..8a9c332b 100644
--- a/lib/philomena/processors/svg.ex
+++ b/lib/philomena/processors/svg.ex
@@ -1,6 +1,12 @@
 defmodule Philomena.Processors.Svg do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    sizes
+    |> Enum.map(fn {name, _} -> "#{name}.png" end)
+    |> Kernel.++(["rendered.png", "full.png"])
+  end
+
   def process(analysis, file, versions) do
     preview = preview(file)
 
diff --git a/lib/philomena/processors/webm.ex b/lib/philomena/processors/webm.ex
index 342135ce..b9da1277 100644
--- a/lib/philomena/processors/webm.ex
+++ b/lib/philomena/processors/webm.ex
@@ -2,6 +2,17 @@ defmodule Philomena.Processors.Webm do
   alias Philomena.Intensities
   import Bitwise
 
+  def versions(sizes) do
+    webm_versions = Enum.map(sizes, fn {name, _} -> "#{name}.webm" end)
+    mp4_versions = Enum.map(sizes, fn {name, _} -> "#{name}.mp4" end)
+    gif_versions =
+      sizes
+      |> Enum.filter(fn {name, _} -> name in [:thumb_tiny, :thumb_small, :thumb] end)
+      |> Enum.map(fn {name, _} -> "#{name}.gif" end)
+
+    webm_versions ++ mp4_versions ++ gif_versions
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     duration = analysis.duration
diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex
index 4dc765ac..effb31a3 100644
--- a/lib/philomena/uploader.ex
+++ b/lib/philomena/uploader.ex
@@ -6,6 +6,7 @@ defmodule Philomena.Uploader do
   alias Philomena.Filename
   alias Philomena.Analyzers
   alias Philomena.Sha512
+  alias ExAws.S3
   import Ecto.Changeset
 
   @doc """
@@ -58,18 +59,15 @@ defmodule Philomena.Uploader do
   in the transaction.
   """
   @spec persist_upload(any(), String.t(), String.t()) :: any()
-
-  # sobelow_skip ["Traversal"]
   def persist_upload(model, file_root, field_name) do
     source = Map.get(model, field(upload_key(field_name)))
     dest = Map.get(model, field(field_name))
     target = Path.join(file_root, dest)
-    dir = Path.dirname(target)
 
-    # Create the target directory if it doesn't exist yet,
-    # then write the file.
-    File.mkdir_p!(dir)
-    File.cp!(source, target)
+    source
+    |> S3.Upload.stream_file()
+    |> S3.upload(bucket(), target, acl: :public_read)
+    |> ExAws.request!()
   end
 
   @doc """
@@ -107,8 +105,11 @@ defmodule Philomena.Uploader do
   defp try_remove("", _file_root), do: nil
   defp try_remove(nil, _file_root), do: nil
 
-  # sobelow_skip ["Traversal.FileModule"]
-  defp try_remove(file, file_root), do: File.rm(Path.join(file_root, file))
+  defp try_remove(file, file_root) do
+    path = Path.join(file_root, file)
+
+    ExAws.request!(S3.delete_object(bucket(), path))
+  end
 
   defp prefix_attributes(map, prefix),
     do: Map.new(map, fn {key, value} -> {"#{prefix}_#{key}", value} end)
@@ -118,4 +119,8 @@ defmodule Philomena.Uploader do
   defp remove_key(field_name), do: "removed_#{field_name}"
 
   defp field(field_name), do: String.to_existing_atom(field_name)
+
+  defp bucket do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
 end

From 1670fd9eb9c7e09591c7f1ec90affc54baece746 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 7 Feb 2022 22:30:39 -0500
Subject: [PATCH 03/39] Remove scale_if_smaller, no more symlinking

---
 lib/philomena/processors/gif.ex  | 30 +++++-------------------------
 lib/philomena/processors/jpeg.ex | 23 ++++-------------------
 lib/philomena/processors/png.ex  | 26 +++-----------------------
 lib/philomena/processors/svg.ex  | 30 ++++++------------------------
 lib/philomena/processors/webm.ex | 31 ++++++-------------------------
 5 files changed, 24 insertions(+), 116 deletions(-)

diff --git a/lib/philomena/processors/gif.ex b/lib/philomena/processors/gif.ex
index 757d6968..8b7557e5 100644
--- a/lib/philomena/processors/gif.ex
+++ b/lib/philomena/processors/gif.ex
@@ -8,18 +8,18 @@ defmodule Philomena.Processors.Gif do
   end
 
   def process(analysis, file, versions) do
-    dimensions = analysis.dimensions
     duration = analysis.duration
     preview = preview(duration, file)
     palette = palette(file)
 
     {:ok, intensities} = Intensities.file(preview)
 
-    scaled = Enum.flat_map(versions, &scale_if_smaller(palette, file, dimensions, &1))
+    scaled = Enum.flat_map(versions, &scale(palette, file, &1))
+    videos = generate_videos(file)
 
     %{
       intensities: intensities,
-      thumbnails: scaled ++ [{:copy, preview, "rendered.png"}]
+      thumbnails: scaled ++ videos ++ [{:copy, preview, "rendered.png"}]
     }
   end
 
@@ -66,27 +66,7 @@ defmodule Philomena.Processors.Gif do
     palette
   end
 
-  # Generate full version, and WebM and MP4 previews
-  defp scale_if_smaller(_palette, file, _dimensions, {:full, _target_dim}) do
-    [{:symlink_original, "full.gif"}] ++ generate_videos(file)
-  end
-
-  defp scale_if_smaller(
-         palette,
-         file,
-         {width, height},
-         {thumb_name, {target_width, target_height}}
-       ) do
-    if width > target_width or height > target_height do
-      scaled = scale(palette, file, {target_width, target_height})
-
-      [{:copy, scaled, "#{thumb_name}.gif"}]
-    else
-      [{:symlink_original, "#{thumb_name}.gif"}]
-    end
-  end
-
-  defp scale(palette, file, {width, height}) do
+  defp scale(palette, file, {thumb_name, {width, height}}) do
     scaled = Briefly.create!(extname: ".gif")
 
     scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
@@ -110,7 +90,7 @@ defmodule Philomena.Processors.Gif do
         scaled
       ])
 
-    scaled
+    [{:copy, scaled, "#{thumb_name}.gif"}]
   end
 
   defp generate_videos(file) do
diff --git a/lib/philomena/processors/jpeg.ex b/lib/philomena/processors/jpeg.ex
index 18257d9f..c3215ec1 100644
--- a/lib/philomena/processors/jpeg.ex
+++ b/lib/philomena/processors/jpeg.ex
@@ -5,13 +5,12 @@ defmodule Philomena.Processors.Jpeg do
     Enum.map(sizes, fn {name, _} -> "#{name}.jpg" end)
   end
 
-  def process(analysis, file, versions) do
-    dimensions = analysis.dimensions
+  def process(_analysis, file, versions) do
     stripped = optimize(strip(file))
 
     {:ok, intensities} = Intensities.file(stripped)
 
-    scaled = Enum.flat_map(versions, &scale_if_smaller(stripped, dimensions, &1))
+    scaled = Enum.flat_map(versions, &scale(stripped, &1))
 
     %{
       replace_original: stripped,
@@ -72,21 +71,7 @@ defmodule Philomena.Processors.Jpeg do
     optimized
   end
 
-  defp scale_if_smaller(_file, _dimensions, {:full, _target_dim}) do
-    [{:symlink_original, "full.jpg"}]
-  end
-
-  defp scale_if_smaller(file, {width, height}, {thumb_name, {target_width, target_height}}) do
-    if width > target_width or height > target_height do
-      scaled = scale(file, {target_width, target_height})
-
-      [{:copy, scaled, "#{thumb_name}.jpg"}]
-    else
-      [{:symlink_original, "#{thumb_name}.jpg"}]
-    end
-  end
-
-  defp scale(file, {width, height}) do
+  defp scale(file, {thumb_name, {width, height}}) do
     scaled = Briefly.create!(extname: ".jpg")
     scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
 
@@ -106,7 +91,7 @@ defmodule Philomena.Processors.Jpeg do
 
     {_output, 0} = System.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled])
 
-    scaled
+    [{:copy, scaled, "#{thumb_name}.jpg"}]
   end
 
   defp srgb_profile do
diff --git a/lib/philomena/processors/png.ex b/lib/philomena/processors/png.ex
index 91d2c87f..373ede0d 100644
--- a/lib/philomena/processors/png.ex
+++ b/lib/philomena/processors/png.ex
@@ -6,12 +6,11 @@ defmodule Philomena.Processors.Png do
   end
 
   def process(analysis, file, versions) do
-    dimensions = analysis.dimensions
     animated? = analysis.animated?
 
     {:ok, intensities} = Intensities.file(file)
 
-    scaled = Enum.flat_map(versions, &scale_if_smaller(file, animated?, dimensions, &1))
+    scaled = Enum.flat_map(versions, &scale(file, animated?, &1))
 
     %{
       intensities: intensities,
@@ -47,26 +46,7 @@ defmodule Philomena.Processors.Png do
     optimized
   end
 
-  defp scale_if_smaller(_file, _animated?, _dimensions, {:full, _target_dim}) do
-    [{:symlink_original, "full.png"}]
-  end
-
-  defp scale_if_smaller(
-         file,
-         animated?,
-         {width, height},
-         {thumb_name, {target_width, target_height}}
-       ) do
-    if width > target_width or height > target_height do
-      scaled = scale(file, animated?, {target_width, target_height})
-
-      [{:copy, scaled, "#{thumb_name}.png"}]
-    else
-      [{:symlink_original, "#{thumb_name}.png"}]
-    end
-  end
-
-  defp scale(file, animated?, {width, height}) do
+  defp scale(file, animated?, {thumb_name, {width, height}}) do
     scaled = Briefly.create!(extname: ".png")
 
     scale_filter =
@@ -96,6 +76,6 @@ defmodule Philomena.Processors.Png do
 
     System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
 
-    scaled
+    [{:copy, scaled, "#{thumb_name}.png"}]
   end
 end
diff --git a/lib/philomena/processors/svg.ex b/lib/philomena/processors/svg.ex
index 8a9c332b..7f45b893 100644
--- a/lib/philomena/processors/svg.ex
+++ b/lib/philomena/processors/svg.ex
@@ -7,16 +7,17 @@ defmodule Philomena.Processors.Svg do
     |> Kernel.++(["rendered.png", "full.png"])
   end
 
-  def process(analysis, file, versions) do
+  def process(_analysis, file, versions) do
     preview = preview(file)
 
     {:ok, intensities} = Intensities.file(preview)
 
-    scaled = Enum.flat_map(versions, &scale_if_smaller(file, analysis.dimensions, preview, &1))
+    scaled = Enum.flat_map(versions, &scale(preview, &1))
+    full = [{:copy, preview, "full.png"}]
 
     %{
       intensities: intensities,
-      thumbnails: scaled ++ [{:copy, preview, "rendered.png"}]
+      thumbnails: scaled ++ full ++ [{:copy, preview, "rendered.png"}]
     }
   end
 
@@ -35,26 +36,7 @@ defmodule Philomena.Processors.Svg do
     preview
   end
 
-  defp scale_if_smaller(_file, _dimensions, preview, {:full, _target_dim}) do
-    [{:symlink_original, "full.svg"}, {:copy, preview, "full.png"}]
-  end
-
-  defp scale_if_smaller(
-         _file,
-         {width, height},
-         preview,
-         {thumb_name, {target_width, target_height}}
-       ) do
-    if width > target_width or height > target_height do
-      scaled = scale(preview, {target_width, target_height})
-
-      [{:copy, scaled, "#{thumb_name}.png"}]
-    else
-      [{:copy, preview, "#{thumb_name}.png"}]
-    end
-  end
-
-  defp scale(preview, {width, height}) do
+  defp scale(preview, {thumb_name, {width, height}}) do
     scaled = Briefly.create!(extname: ".png")
     scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
 
@@ -63,6 +45,6 @@ defmodule Philomena.Processors.Svg do
 
     {_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
 
-    scaled
+    [{:copy, scaled, "#{thumb_name}.png"}]
   end
 end
diff --git a/lib/philomena/processors/webm.ex b/lib/philomena/processors/webm.ex
index b9da1277..3fd76cc1 100644
--- a/lib/philomena/processors/webm.ex
+++ b/lib/philomena/processors/webm.ex
@@ -23,13 +23,13 @@ defmodule Philomena.Processors.Webm do
 
     {:ok, intensities} = Intensities.file(preview)
 
-    scaled =
-      Enum.flat_map(versions, &scale_if_smaller(stripped, mp4, palette, duration, dimensions, &1))
+    scaled = Enum.flat_map(versions, &scale(stripped, palette, duration, dimensions, &1))
+    mp4 = [{:copy, mp4, "full.mp4"}]
 
     %{
       replace_original: stripped,
       intensities: intensities,
-      thumbnails: scaled ++ [{:copy, preview, "rendered.png"}]
+      thumbnails: scaled ++ mp4 ++ [{:copy, preview, "rendered.png"}]
     }
   end
 
@@ -70,31 +70,12 @@ defmodule Philomena.Processors.Webm do
     stripped
   end
 
-  defp scale_if_smaller(_file, mp4, _palette, _duration, _dimensions, {:full, _target_dim}) do
-    [
-      {:symlink_original, "full.webm"},
-      {:copy, mp4, "full.mp4"}
-    ]
-  end
-
-  defp scale_if_smaller(
-         file,
-         mp4,
-         palette,
-         duration,
-         {width, height},
-         {thumb_name, {target_width, target_height}}
-       ) do
-    {webm, mp4} =
-      if width > target_width or height > target_height do
-        scale_videos(file, {width, height}, {target_width, target_height})
-      else
-        {file, mp4}
-      end
+  defp scale(file, palette, duration, dimensions, {thumb_name, target_dimensions}) do
+    {webm, mp4} = scale_videos(file, dimensions, target_dimensions)
 
     cond do
       thumb_name in [:thumb, :thumb_small, :thumb_tiny] ->
-        gif = scale_gif(file, palette, duration, {target_width, target_height})
+        gif = scale_gif(file, palette, duration, target_dimensions)
 
         [
           {:copy, webm, "#{thumb_name}.webm"},

From 4c97791a7b1b2a0afef0f917b0a8cbf2a8c3c9bb Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 7 Feb 2022 22:41:59 -0500
Subject: [PATCH 04/39] Compatible thumbnailer

---
 lib/philomena/images/thumbnailer.ex | 99 +++++------------------------
 1 file changed, 17 insertions(+), 82 deletions(-)

diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index c1c8cdeb..8ea1da72 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -39,7 +39,7 @@ defmodule Philomena.Images.Thumbnailer do
     file = download_image_file(image)
     {:ok, analysis} = Analyzers.analyze(file)
 
-    apply_edit_script(image, Processors.process(analysis, file, @versions))
+    apply_edit_script(image, Processors.process(analysis, file, generated_sizes(image)))
     generate_dupe_reports(image)
     recompute_meta(image, file, &Image.thumbnail_changeset/2)
 
@@ -54,16 +54,13 @@ defmodule Philomena.Images.Thumbnailer do
     do: ImageIntensities.create_image_intensity(image, intensities)
 
   defp apply_change(image, {:replace_original, new_file}),
-    do: copy(new_file, image_file(image))
+    do: upload_file(image, new_file, "full.#{image.image_format}")
 
   defp apply_change(image, {:thumbnails, thumbnails}),
-    do: Enum.map(thumbnails, &apply_thumbnail(image, image_thumb_dir(image), &1))
+    do: Enum.map(thumbnails, &apply_thumbnail(image, &1))
 
-  defp apply_thumbnail(_image, thumb_dir, {:copy, new_file, destination}),
-    do: copy(new_file, Path.join(thumb_dir, destination))
-
-  defp apply_thumbnail(image, thumb_dir, {:symlink_original, destination}),
-    do: symlink(image_file(image), Path.join(thumb_dir, destination))
+  defp apply_thumbnail(image, {:copy, new_file, destination}),
+    do: upload_file(image, new_file, destination)
 
   defp generate_dupe_reports(image) do
     if not image.duplication_checked do
@@ -84,103 +81,41 @@ defmodule Philomena.Images.Thumbnailer do
     |> Repo.update!()
   end
 
-  # Copy from source to destination, creating parent directories along
-  # the way and setting the appropriate permission bits when necessary.
-  #
-  # sobelow_skip ["Traversal.FileModule"]
-  defp copy(source, destination) do
-    prepare_dir(destination)
-
-    File.rm(destination)
-    File.cp!(source, destination)
-
-    set_perms(destination)
-  end
-
-  # Try to handle filesystems that don't support symlinks
-  # by falling back to a copy.
-  #
-  # sobelow_skip ["Traversal.FileModule"]
-  defp symlink(source, destination) do
-    source = Path.absname(source)
-
-    prepare_dir(destination)
-
-    case File.ln_s(source, destination) do
-      :ok ->
-        set_perms(destination)
-
-      _err ->
-        copy(source, destination)
-    end
-  end
-
-  # 0o644 = (S_IRUSR | S_IWUSR) | S_IRGRP | S_IROTH
-  #
-  # sobelow_skip ["Traversal.FileModule"]
-  defp set_perms(destination),
-    do: File.chmod(destination, 0o644)
-
-  # Prepare the directory by creating it if it does not yet exist.
-  #
-  # sobelow_skip ["Traversal.FileModule"]
-  defp prepare_dir(destination) do
-    destination
-    |> Path.dirname()
-    |> File.mkdir_p!()
-  end
-
-  def image_file(%Image{image: image}),
-    do: Path.join(image_file_root(), image)
-
-  defp download_image_file(%Image{image: path} = image) do
+  defp download_image_file(image) do
     tempfile = Briefly.create!(extname: "." <> image.image_format)
-    path = Path.join(image_file_root(), path)
+    path = Path.join(image_thumb_prefix(image), "full.#{image.image_format}")
 
     ExAws.request!(S3.download_file(bucket(), path, tempfile))
 
     tempfile
   end
 
-  defp upload_image_file(%Image{image: path}, new_file) do
-    path = Path.join(image_file_root(), path)
+  defp upload_file(image, file, version_name) do
+    path = Path.join(image_thumb_prefix(image), version_name)
 
-    new_file
+    file
     |> S3.Upload.stream_file()
     |> S3.upload(bucket(), path, acl: :public_read)
     |> ExAws.request!()
   end
 
-  def image_thumb_dir(%Image{
+  defp image_thumb_prefix(%Image{
         created_at: created_at,
         id: id,
         hidden_from_users: true,
         hidden_image_key: key
       }),
-      do: Path.join([image_thumbnail_root(), time_identifier(created_at), "#{id}-#{key}"])
+      do: Path.join([image_file_root(), time_identifier(created_at), "#{id}-#{key}"])
 
-  def image_thumb_dir(%Image{created_at: created_at, id: id}),
-    do: Path.join([image_thumbnail_root(), time_identifier(created_at), to_string(id)])
-
-  defp image_url_base(%Image{created_at: created_at, id: id}, nil),
-    do: Path.join([image_url_root(), time_identifier(created_at), to_string(id)])
-
-  defp image_url_base(%Image{created_at: created_at, id: id}, key),
-    do: Path.join([image_url_root(), time_identifier(created_at), "#{id}-#{key}"])
+  defp image_thumb_prefix(%Image{created_at: created_at, id: id}),
+    do: Path.join([image_file_root(), time_identifier(created_at), to_string(id)])
 
   defp time_identifier(time),
     do: Enum.join([time.year, time.month, time.day], "/")
 
   defp image_file_root,
-    do: Application.get_env(:philomena, :image_file_root)
+    do: Application.fetch_env!(:philomena, :image_file_root)
 
-  defp image_thumbnail_root,
-    do: Application.get_env(:philomena, :image_file_root) <> "/thumbs"
-
-  defp image_url_root,
-    do: Application.get_env(:philomena, :image_url_root)
-
-  defp bucket() do
-    Application.fetch_env!(:philomena, :s3_bucket)
-  end
+  defp bucket,
+    do: Application.fetch_env!(:philomena, :s3_bucket)
 end

From 20cc6cb9088e6927e16277e29088622948c0b222 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 7 Feb 2022 22:58:52 -0500
Subject: [PATCH 05/39] Finish thumbnailer, integrate hider

---
 lib/philomena/images/hider.ex       | 54 ---------------------
 lib/philomena/images/thumbnailer.ex | 74 +++++++++++++++++++++++++----
 lib/philomena/processors.ex         |  8 ++--
 3 files changed, 68 insertions(+), 68 deletions(-)
 delete mode 100644 lib/philomena/images/hider.ex

diff --git a/lib/philomena/images/hider.ex b/lib/philomena/images/hider.ex
deleted file mode 100644
index aba0100d..00000000
--- a/lib/philomena/images/hider.ex
+++ /dev/null
@@ -1,54 +0,0 @@
-defmodule Philomena.Images.Hider do
-  @moduledoc """
-  Hiding logic for images.
-  """
-
-  alias Philomena.Images.Image
-
-  # sobelow_skip ["Traversal.FileModule"]
-  def hide_thumbnails(image, key) do
-    source = image_thumb_dir(image)
-    target = image_thumb_dir(image, key)
-
-    File.rm_rf(target)
-    File.rename(source, target)
-  end
-
-  # sobelow_skip ["Traversal.FileModule"]
-  def unhide_thumbnails(image, key) do
-    source = image_thumb_dir(image, key)
-    target = image_thumb_dir(image)
-
-    File.rm_rf(target)
-    File.rename(source, target)
-  end
-
-  # sobelow_skip ["Traversal.FileModule"]
-  def destroy_thumbnails(image) do
-    hidden = image_thumb_dir(image, image.hidden_image_key)
-    normal = image_thumb_dir(image)
-
-    File.rm_rf(hidden)
-    File.rm_rf(normal)
-  end
-
-  def purge_cache(files) do
-    {_out, 0} = System.cmd("purge-cache", [Jason.encode!(%{files: files})])
-
-    :ok
-  end
-
-  # fixme: these are copied from the thumbnailer
-  defp image_thumb_dir(%Image{created_at: created_at, id: id}),
-    do: Path.join([image_thumbnail_root(), time_identifier(created_at), to_string(id)])
-
-  defp image_thumb_dir(%Image{created_at: created_at, id: id}, key),
-    do:
-      Path.join([image_thumbnail_root(), time_identifier(created_at), to_string(id) <> "-" <> key])
-
-  defp time_identifier(time),
-    do: Enum.join([time.year, time.month, time.day], "/")
-
-  defp image_thumbnail_root,
-    do: Application.get_env(:philomena, :image_file_root) <> "/thumbs"
-end
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 8ea1da72..2a14a4cb 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -22,6 +22,8 @@ defmodule Philomena.Images.Thumbnailer do
     tall: {1024, 4096}
   ]
 
+  @acl [acl: :public_read]
+
   def thumbnail_versions do
     @versions
   end
@@ -34,6 +36,34 @@ defmodule Philomena.Images.Thumbnailer do
     end)
   end
 
+  def hide_thumbnails(image, key) do
+    moved_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+
+    source_prefix = visible_image_thumb_prefix(image)
+    target_prefix = hidden_image_thumb_prefix(image, key)
+
+    bulk_rename(moved_files, source_prefix, target_prefix)
+  end
+
+  def unhide_thumbnails(image, key) do
+    moved_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+
+    source_prefix = hidden_image_thumb_prefix(image, key)
+    target_prefix = visible_image_thumb_prefix(image)
+
+    bulk_rename(moved_files, source_prefix, target_prefix)
+  end
+
+  def destroy_thumbnails(image) do
+    affected_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+
+    hidden_prefix = hidden_image_thumb_prefix(image, image.hidden_image_key)
+    visible_prefix = visible_image_thumb_prefix(image)
+
+    bulk_delete(affected_files, hidden_prefix)
+    bulk_delete(affected_files, visible_prefix)
+  end
+
   def generate_thumbnails(image_id) do
     image = Repo.get!(Image, image_id)
     file = download_image_file(image)
@@ -82,7 +112,7 @@ defmodule Philomena.Images.Thumbnailer do
   end
 
   defp download_image_file(image) do
-    tempfile = Briefly.create!(extname: "." <> image.image_format)
+    tempfile = Briefly.create!(extname: ".#{image.image_format}")
     path = Path.join(image_thumb_prefix(image), "full.#{image.image_format}")
 
     ExAws.request!(S3.download_file(bucket(), path, tempfile))
@@ -95,19 +125,43 @@ defmodule Philomena.Images.Thumbnailer do
 
     file
     |> S3.Upload.stream_file()
-    |> S3.upload(bucket(), path, acl: :public_read)
+    |> S3.upload(bucket(), path, @acl)
     |> ExAws.request!()
   end
 
-  defp image_thumb_prefix(%Image{
-        created_at: created_at,
-        id: id,
-        hidden_from_users: true,
-        hidden_image_key: key
-      }),
-      do: Path.join([image_file_root(), time_identifier(created_at), "#{id}-#{key}"])
+  def bulk_rename(file_names, source_prefix, target_prefix) do
+    Enum.map(file_names, fn name ->
+      source = Path.join(source_prefix, name)
+      target = Path.join(target_prefix, name)
 
-  defp image_thumb_prefix(%Image{created_at: created_at, id: id}),
+      ExAws.request!(S3.put_object_copy(bucket(), target, bucket(), source, @acl))
+      ExAws.request!(S3.delete_object(bucket(), source))
+    end)
+  end
+
+  def bulk_delete(file_names, prefix) do
+    Enum.map(file_names, fn name ->
+      target = Path.join(prefix, name)
+
+      ExAws.request!(S3.delete_object(bucket(), target))
+    end)
+  end
+
+  # This method wraps the following two for code that doesn't care
+  # and just wants the files (most code should take this path)
+
+  defp image_thumb_prefix(%{hidden_from_users: true} = image),
+    do: hidden_image_thumb_prefix(image, image.hidden_image_key)
+
+  defp image_thumb_prefix(image),
+    do: visible_image_thumb_prefix(image)
+
+  # These methods handle the actual distinction between the two
+
+  defp hidden_image_thumb_prefix(%Image{created_at: created_at, id: id}, key),
+    do: Path.join([image_file_root(), time_identifier(created_at), "#{id}-#{key}"])
+
+  defp visible_image_thumb_prefix(%Image{created_at: created_at, id: id}),
     do: Path.join([image_file_root(), time_identifier(created_at), to_string(id)])
 
   defp time_identifier(time),
diff --git a/lib/philomena/processors.ex b/lib/philomena/processors.ex
index 531d0533..202da1d4 100644
--- a/lib/philomena/processors.ex
+++ b/lib/philomena/processors.ex
@@ -41,12 +41,12 @@ defmodule Philomena.Processors do
   def processor(_content_type), do: nil
 
   @doc """
-  Takes an analyzer and version list and generates a list of versions to be
+  Takes a MIME type and version list and generates a list of versions to be
   generated (e.g., ["thumb.png"]). List contents differ based on file type.
   """
-  @spec versions(map(), keyword) :: [String.t()]
-  def versions(analysis, valid_sizes) do
-    processor(analysis.mime_type).versions(valid_sizes)
+  @spec versions(String.t(), keyword) :: [String.t()]
+  def versions(mime_type, valid_sizes) do
+    processor(mime_type).versions(valid_sizes)
   end
 
   @doc """

From f80d8577a837fc14df3928fc816e941579ad553e Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Tue, 8 Feb 2022 19:08:53 -0500
Subject: [PATCH 06/39] Finish cleanup of thumbnailer

---
 lib/mix/tasks/recalculate_intensities.ex | 53 ------------------------
 lib/philomena/filename.ex                | 16 +------
 lib/philomena/images.ex                  | 11 ++---
 lib/philomena/images/thumbnailer.ex      | 18 +++++++-
 lib/philomena/images/uploader.ex         |  9 ++--
 5 files changed, 27 insertions(+), 80 deletions(-)
 delete mode 100644 lib/mix/tasks/recalculate_intensities.ex

diff --git a/lib/mix/tasks/recalculate_intensities.ex b/lib/mix/tasks/recalculate_intensities.ex
deleted file mode 100644
index ad3ee171..00000000
--- a/lib/mix/tasks/recalculate_intensities.ex
+++ /dev/null
@@ -1,53 +0,0 @@
-defmodule Mix.Tasks.RecalculateIntensities do
-  use Mix.Task
-
-  alias Philomena.Images.{Image, Thumbnailer}
-  alias Philomena.ImageIntensities.ImageIntensity
-  alias Philomena.Batch
-  alias Philomena.Repo
-
-  import Ecto.Query
-
-  @shortdoc "Recalculates all intensities for reverse search."
-  @requirements ["app.start"]
-  @impl Mix.Task
-  def run(_args) do
-    Batch.record_batches(Image, fn batch ->
-      batch
-      |> Stream.with_index()
-      |> Stream.each(fn {image, i} ->
-        image_file =
-          cond do
-            image.image_mime_type in ["image/png", "image/jpeg"] ->
-              Thumbnailer.image_file(image)
-
-            true ->
-              Path.join(Thumbnailer.image_thumb_dir(image), "rendered.png")
-          end
-
-        case System.cmd("image-intensities", [image_file]) do
-          {output, 0} ->
-            [nw, ne, sw, se] =
-              output
-              |> String.trim()
-              |> String.split("\t")
-              |> Enum.map(&String.to_float/1)
-
-            ImageIntensity
-            |> where(image_id: ^image.id)
-            |> Repo.update_all(set: [nw: nw, ne: ne, sw: sw, se: se])
-
-          _ ->
-            :err
-        end
-
-        if rem(i, 100) == 0 do
-          IO.write("\r#{image.id}")
-        end
-      end)
-      |> Stream.run()
-    end)
-
-    IO.puts("\nDone")
-  end
-end
diff --git a/lib/philomena/filename.ex b/lib/philomena/filename.ex
index 43b081a1..ea0d230d 100644
--- a/lib/philomena/filename.ex
+++ b/lib/philomena/filename.ex
@@ -8,8 +8,7 @@ defmodule Philomena.Filename do
     [
       time_identifier(DateTime.utc_now()),
       "/",
-      usec_identifier(),
-      pid_identifier(),
+      UUID.uuid1(),
       ".",
       extension
     ]
@@ -19,17 +18,4 @@ defmodule Philomena.Filename do
   defp time_identifier(time) do
     Enum.join([time.year, time.month, time.day], "/")
   end
-
-  defp usec_identifier do
-    DateTime.utc_now()
-    |> DateTime.to_unix(:microsecond)
-    |> to_string()
-  end
-
-  defp pid_identifier do
-    self()
-    |> :erlang.pid_to_list()
-    |> to_string()
-    |> String.replace(~r/[^0-9]/, "")
-  end
 end
diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex
index 9feda6ab..a244f280 100644
--- a/lib/philomena/images.ex
+++ b/lib/philomena/images.ex
@@ -13,7 +13,6 @@ defmodule Philomena.Images do
   alias Philomena.ImagePurgeWorker
   alias Philomena.DuplicateReports.DuplicateReport
   alias Philomena.Images.Image
-  alias Philomena.Images.Hider
   alias Philomena.Images.Uploader
   alias Philomena.Images.Tagging
   alias Philomena.Images.Thumbnailer
@@ -198,7 +197,7 @@ defmodule Philomena.Images do
       {:ok, image} ->
         Uploader.unpersist_old_upload(image)
         purge_files(image, image.hidden_image_key)
-        Hider.destroy_thumbnails(image)
+        Thumbnailer.destroy_thumbnails(image)
 
         {:ok, image}
 
@@ -539,7 +538,7 @@ defmodule Philomena.Images do
   defp process_after_hide(result) do
     case result do
       {:ok, %{image: image, tags: tags, reports: {_count, reports}} = result} ->
-        Hider.hide_thumbnails(image, image.hidden_image_key)
+        Thumbnailer.hide_thumbnails(image, image.hidden_image_key)
 
         Comments.reindex_comments(image)
         Reports.reindex_reports(reports)
@@ -590,7 +589,7 @@ defmodule Philomena.Images do
     |> Repo.transaction()
     |> case do
       {:ok, %{image: image, tags: tags}} ->
-        Hider.unhide_thumbnails(image, key)
+        Thumbnailer.unhide_thumbnails(image, key)
 
         reindex_image(image)
         purge_files(image, image.hidden_image_key)
@@ -774,7 +773,9 @@ defmodule Philomena.Images do
   end
 
   def perform_purge(files) do
-    Hider.purge_cache(files)
+    {_out, 0} = System.cmd("purge-cache", [Jason.encode!(%{files: files})])
+
+    :ok
   end
 
   alias Philomena.Images.Subscription
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 2a14a4cb..f078ddf1 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -36,6 +36,13 @@ defmodule Philomena.Images.Thumbnailer do
     end)
   end
 
+  def thumbnail_urls(image, hidden_key) do
+    Processors.versions(image.image_mime_type, generated_sizes(image))
+    |> Enum.map(fn name ->
+      Path.join(image_url_base(image, hidden_key), name)
+    end)
+  end
+
   def hide_thumbnails(image, key) do
     moved_files = Processors.versions(image.image_mime_type, generated_sizes(image))
 
@@ -120,7 +127,7 @@ defmodule Philomena.Images.Thumbnailer do
     tempfile
   end
 
-  defp upload_file(image, file, version_name) do
+  def upload_file(image, file, version_name) do
     path = Path.join(image_thumb_prefix(image), version_name)
 
     file
@@ -164,12 +171,21 @@ defmodule Philomena.Images.Thumbnailer do
   defp visible_image_thumb_prefix(%Image{created_at: created_at, id: id}),
     do: Path.join([image_file_root(), time_identifier(created_at), to_string(id)])
 
+  defp image_url_base(%Image{created_at: created_at, id: id}, nil),
+    do: Path.join([image_url_root(), time_identifier(created_at), to_string(id)])
+
+  defp image_url_base(%Image{created_at: created_at, id: id}, key),
+    do: Path.join([image_url_root(), time_identifier(created_at), "#{id}-#{key}"])
+
   defp time_identifier(time),
     do: Enum.join([time.year, time.month, time.day], "/")
 
   defp image_file_root,
     do: Application.fetch_env!(:philomena, :image_file_root)
 
+  defp image_url_root,
+    do: Application.fetch_env!(:philomena, :image_url_root)
+
   defp bucket,
     do: Application.fetch_env!(:philomena, :s3_bucket)
 end
diff --git a/lib/philomena/images/uploader.ex b/lib/philomena/images/uploader.ex
index 35b96394..8419b805 100644
--- a/lib/philomena/images/uploader.ex
+++ b/lib/philomena/images/uploader.ex
@@ -3,6 +3,7 @@ defmodule Philomena.Images.Uploader do
   Upload and processing callback logic for Images.
   """
 
+  alias Philomena.Images.Thumbnailer
   alias Philomena.Images.Image
   alias Philomena.Uploader
 
@@ -11,14 +12,10 @@ defmodule Philomena.Images.Uploader do
   end
 
   def persist_upload(image) do
-    Uploader.persist_upload(image, image_file_root(), "image")
+    Thumbnailer.upload_file(image, image.uploaded_image, "full.#{image.image_format}")
   end
 
   def unpersist_old_upload(image) do
-    Uploader.unpersist_old_upload(image, image_file_root(), "image")
-  end
-
-  defp image_file_root do
-    Application.get_env(:philomena, :image_file_root)
+    Thumbnailer.destroy_thumbnails(image)
   end
 end

From 74f34a18808303d05797317ab4bad3b1b8c27c5d Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Tue, 8 Feb 2022 19:17:24 -0500
Subject: [PATCH 07/39] Run app

---
 config/runtime.exs                   | 2 +-
 docker-compose.yml                   | 2 +-
 docker/files/run-docker-container.sh | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 mode change 100644 => 100755 docker/files/run-docker-container.sh

diff --git a/config/runtime.exs b/config/runtime.exs
index 9605d362..46a143b8 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -31,7 +31,7 @@ config :philomena,
   tag_url_root: System.fetch_env!("TAG_URL_ROOT"),
   redis_host: System.get_env("REDIS_HOST", "localhost"),
   proxy_host: System.get_env("PROXY_HOST"),
-  s3_bucket: System.fetch_env!("S3_BUCKET")
+  s3_bucket: System.fetch_env!("S3_BUCKET"),
   camo_host: System.get_env("CAMO_HOST"),
   camo_key: System.get_env("CAMO_KEY"),
   cdn_host: System.fetch_env!("CDN_HOST")
diff --git a/docker-compose.yml b/docker-compose.yml
index 9a27584b..bb30a25d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -79,7 +79,7 @@ services:
   files:
     build:
       context: .
-      dockerfile: ./docker/app/Dockerfile
+      dockerfile: ./docker/files/Dockerfile
     volumes:
       - .:/srv/philomena
 
diff --git a/docker/files/run-docker-container.sh b/docker/files/run-docker-container.sh
old mode 100644
new mode 100755

From 577d5e8766b7f0297a99077a296ffc8060dcf68d Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Tue, 8 Feb 2022 19:17:39 -0500
Subject: [PATCH 08/39] Format

---
 lib/philomena/processors/webm.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/philomena/processors/webm.ex b/lib/philomena/processors/webm.ex
index 3fd76cc1..9803b54e 100644
--- a/lib/philomena/processors/webm.ex
+++ b/lib/philomena/processors/webm.ex
@@ -5,6 +5,7 @@ defmodule Philomena.Processors.Webm do
   def versions(sizes) do
     webm_versions = Enum.map(sizes, fn {name, _} -> "#{name}.webm" end)
     mp4_versions = Enum.map(sizes, fn {name, _} -> "#{name}.mp4" end)
+
     gif_versions =
       sizes
       |> Enum.filter(fn {name, _} -> name in [:thumb_tiny, :thumb_small, :thumb] end)

From 963c20c60c05f696ff88dff1c9d372e25f069895 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Tue, 8 Feb 2022 21:32:12 -0500
Subject: [PATCH 09/39] nginx configuration to serve files

---
 docker/app/run-development | 10 +++++-----
 docker/web/Dockerfile      |  3 ++-
 docker/web/nginx.conf      | 30 ++++++++++++++++++++----------
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/docker/app/run-development b/docker/app/run-development
index 5e171745..90a315d6 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -2,11 +2,11 @@
 
 # Create S3 dirs
 mkdir -p /srv/philomena/priv/s3/philomena
-ln -s /srv/philomena/priv/system/images/thumbs /srv/philomena/priv/s3/philomena/images
-ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/adverts
-ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/avatars
-ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/badges
-ln -s /srv/philomena/priv/system/images /srv/philomena/priv/s3/philomena/tags
+ln -sf /srv/philomena/priv/static/system/images/thumbs /srv/philomena/priv/s3/philomena/images
+ln -sf /srv/philomena/priv/static/system/images /srv/philomena/priv/s3/philomena/adverts
+ln -sf /srv/philomena/priv/static/system/images /srv/philomena/priv/s3/philomena/avatars
+ln -sf /srv/philomena/priv/static/system/images /srv/philomena/priv/s3/philomena/badges
+ln -sf /srv/philomena/priv/static/system/images /srv/philomena/priv/s3/philomena/tags
 
 # For compatibility with musl libc
 export CARGO_FEATURE_DISABLE_INITIAL_EXEC_TLS=1
diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile
index bf79c468..e76dab63 100644
--- a/docker/web/Dockerfile
+++ b/docker/web/Dockerfile
@@ -1,6 +1,7 @@
 FROM nginx:1.21.4-alpine
 ENV APP_DIR /srv/philomena
+ENV BUCKET_NAME philomena
 COPY docker/web/nginx.conf /tmp/docker.nginx
-RUN envsubst '$APP_DIR' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf
+RUN envsubst '$APP_DIR $BUCKET_NAME' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf
 EXPOSE 80
 CMD ["nginx", "-g", "daemon off;"]
diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index c7e9f26d..012d6fa6 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -2,6 +2,10 @@ upstream philomena {
 	server app:4000 fail_timeout=0;
 }
 
+upstream s3 {
+	server files:80 fail_timeout=0;
+}
+
 server {
 	listen 80 default;
 	listen [::]:80;
@@ -14,38 +18,44 @@ server {
 	location ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
 		expires max;
 		add_header      Cache-Control public;
-		alias "$APP_DIR/priv/static/system/images/thumbs/$1/$2/full.$3";
+		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
 	}
 
 	location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
 		add_header Content-Disposition "attachment";
 		expires max;
 		add_header      Cache-Control public;
-		alias "$APP_DIR/priv/static/system/images/thumbs/$1/$2/full.$3";
+		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
 	}
 
-	location ~ ^/img/(.+)   {
+	location ~ ^/img/(.+) {
 		expires max;
 		add_header Cache-Control public;
-		alias $APP_DIR/priv/static/system/images/thumbs/$1;
+		proxy_pass http://s3/$BUCKET_NAME/images/$1;
 	}
 
-	location ~ ^/spns/(.+)   {
+	location ~ ^/spns/(.+) {
 		expires max;
 		add_header Cache-Control public;
-		alias $APP_DIR/priv/static/system/images/adverts/$1;
+		proxy_pass http://s3/$BUCKET_NAME/adverts/$1;
 	}
 
-	location ~ ^/avatars/(.+)   {
+	location ~ ^/avatars/(.+) {
 		expires max;
 		add_header Cache-Control public;
-		alias $APP_DIR/priv/static/system/images/avatars/$1;
+		proxy_pass http://s3/$BUCKET_NAME/avatars/$1;
 	}
 
-	location ~ ^/media/(.+)   {
+	location ~ ^/badges/(.+) {
 		expires max;
 		add_header Cache-Control public;
-		alias $APP_DIR/priv/static/system/images/$1;
+		proxy_pass http://s3/$BUCKET_NAME/badges/$1;
+	}
+
+	location ~ ^/tags/(.+) {
+		expires max;
+		add_header Cache-Control public;
+		proxy_pass http://s3/$BUCKET_NAME/tags/$1;
 	}
 
 	location / {

From 82c84fcccb17d4e80f5a94f12e4ef557cd3f55ec Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Wed, 9 Feb 2022 20:33:10 -0500
Subject: [PATCH 10/39] Update to latest S3Proxy

---
 docker-compose.yml                   |  6 +++---
 docker/files/Dockerfile              |  2 --
 docker/files/run-docker-container.sh | 28 ----------------------------
 3 files changed, 3 insertions(+), 33 deletions(-)
 delete mode 100644 docker/files/Dockerfile
 delete mode 100755 docker/files/run-docker-container.sh

diff --git a/docker-compose.yml b/docker-compose.yml
index bb30a25d..88443e52 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -77,9 +77,9 @@ services:
       driver: "none"
 
   files:
-    build:
-      context: .
-      dockerfile: ./docker/files/Dockerfile
+    image: andrewgaul/s3proxy:sha-5aec5c1
+    environment:
+      - JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3
     volumes:
       - .:/srv/philomena
 
diff --git a/docker/files/Dockerfile b/docker/files/Dockerfile
deleted file mode 100644
index 9316f2c1..00000000
--- a/docker/files/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM andrewgaul/s3proxy:sha-2e61c38
-COPY docker/files/run-docker-container.sh /opt/s3proxy/run-docker-container.sh
diff --git a/docker/files/run-docker-container.sh b/docker/files/run-docker-container.sh
deleted file mode 100755
index 171cbd55..00000000
--- a/docker/files/run-docker-container.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/sh
-# forked because it doesn't let me configure the filesystem basedir via env
-mkdir -p /srv/philomena/priv/s3
-
-exec java \
-    -DLOG_LEVEL="${LOG_LEVEL}" \
-    -Ds3proxy.endpoint="${S3PROXY_ENDPOINT}" \
-    -Ds3proxy.virtual-host="${S3PROXY_VIRTUALHOST}" \
-    -Ds3proxy.authorization="${S3PROXY_AUTHORIZATION}" \
-    -Ds3proxy.identity="${S3PROXY_IDENTITY}" \
-    -Ds3proxy.credential="${S3PROXY_CREDENTIAL}" \
-    -Ds3proxy.cors-allow-all="${S3PROXY_CORS_ALLOW_ALL}" \
-    -Ds3proxy.cors-allow-origins="${S3PROXY_CORS_ALLOW_ORIGINS}" \
-    -Ds3proxy.cors-allow-methods="${S3PROXY_CORS_ALLOW_METHODS}" \
-    -Ds3proxy.cors-allow-headers="${S3PROXY_CORS_ALLOW_HEADERS}" \
-    -Ds3proxy.ignore-unknown-headers="${S3PROXY_IGNORE_UNKNOWN_HEADERS}" \
-    -Djclouds.provider="${JCLOUDS_PROVIDER}" \
-    -Djclouds.identity="${JCLOUDS_IDENTITY}" \
-    -Djclouds.credential="${JCLOUDS_CREDENTIAL}" \
-    -Djclouds.endpoint="${JCLOUDS_ENDPOINT}" \
-    -Djclouds.region="${JCLOUDS_REGION}" \
-    -Djclouds.regions="${JCLOUDS_REGIONS}" \
-    -Djclouds.keystone.version="${JCLOUDS_KEYSTONE_VERSION}" \
-    -Djclouds.keystone.scope="${JCLOUDS_KEYSTONE_SCOPE}" \
-    -Djclouds.keystone.project-domain-name="${JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME}" \
-    -Djclouds.filesystem.basedir="/srv/philomena/priv/s3" \
-    -jar /opt/s3proxy/s3proxy \
-    --properties /dev/null

From 7094f14072f892cdbd829e27bc7a9f7ca48a6f92 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 10 Feb 2022 18:39:36 -0500
Subject: [PATCH 11/39] Move all affected versions

---
 lib/philomena/images/thumbnailer.ex | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index f078ddf1..2f4236de 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -37,14 +37,15 @@ defmodule Philomena.Images.Thumbnailer do
   end
 
   def thumbnail_urls(image, hidden_key) do
-    Processors.versions(image.image_mime_type, generated_sizes(image))
+    image
+    |> all_versions()
     |> Enum.map(fn name ->
       Path.join(image_url_base(image, hidden_key), name)
     end)
   end
 
   def hide_thumbnails(image, key) do
-    moved_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+    moved_files = all_versions(image)
 
     source_prefix = visible_image_thumb_prefix(image)
     target_prefix = hidden_image_thumb_prefix(image, key)
@@ -53,7 +54,7 @@ defmodule Philomena.Images.Thumbnailer do
   end
 
   def unhide_thumbnails(image, key) do
-    moved_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+    moved_files = all_versions(image)
 
     source_prefix = hidden_image_thumb_prefix(image, key)
     target_prefix = visible_image_thumb_prefix(image)
@@ -62,7 +63,7 @@ defmodule Philomena.Images.Thumbnailer do
   end
 
   def destroy_thumbnails(image) do
-    affected_files = Processors.versions(image.image_mime_type, generated_sizes(image))
+    affected_files = all_versions(image)
 
     hidden_prefix = hidden_image_thumb_prefix(image, image.hidden_image_key)
     visible_prefix = visible_image_thumb_prefix(image)
@@ -136,24 +137,31 @@ defmodule Philomena.Images.Thumbnailer do
     |> ExAws.request!()
   end
 
-  def bulk_rename(file_names, source_prefix, target_prefix) do
+  defp bulk_rename(file_names, source_prefix, target_prefix) do
     Enum.map(file_names, fn name ->
       source = Path.join(source_prefix, name)
       target = Path.join(target_prefix, name)
 
-      ExAws.request!(S3.put_object_copy(bucket(), target, bucket(), source, @acl))
-      ExAws.request!(S3.delete_object(bucket(), source))
+      ExAws.request(S3.put_object_copy(bucket(), target, bucket(), source, @acl))
+      ExAws.request(S3.delete_object(bucket(), source))
     end)
   end
 
-  def bulk_delete(file_names, prefix) do
+  defp bulk_delete(file_names, prefix) do
     Enum.map(file_names, fn name ->
       target = Path.join(prefix, name)
 
-      ExAws.request!(S3.delete_object(bucket(), target))
+      ExAws.request(S3.delete_object(bucket(), target))
     end)
   end
 
+  defp all_versions(image) do
+    generated = Processors.versions(image.image_mime_type, generated_sizes(image))
+    full = ["full.#{image.image_format}"]
+
+    generated ++ full
+  end
+
   # This method wraps the following two for code that doesn't care
   # and just wants the files (most code should take this path)
 

From fd758b7f0d71243def5d50c5751ead182d814a49 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 10 Feb 2022 19:13:14 -0500
Subject: [PATCH 12/39] Deal with content type screwery

---
 docker/web/nginx.conf               | 24 ++++++++++++++++++++++++
 lib/philomena/images/thumbnailer.ex |  6 ++----
 lib/philomena/uploader.ex           | 15 +++++++++++++--
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index 012d6fa6..969cb6e1 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -6,6 +6,16 @@ upstream s3 {
 	server files:80 fail_timeout=0;
 }
 
+map $uri $custom_content_type {
+	default		"text/html";
+	~(.*\.png)$	"image/png";
+	~(.*\.jpe?g)$	"image/jpeg";
+	~(.*\.gif)$	"image/gif";
+	~(.*\.svg)$	"image/svg+xml";
+	~(.*\.mp4)$	"video/mp4";
+	~(.*\.webm)$	"video/webm";
+}
+
 server {
 	listen 80 default;
 	listen [::]:80;
@@ -19,6 +29,8 @@ server {
 		expires max;
 		add_header      Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
@@ -26,36 +38,48 @@ server {
 		expires max;
 		add_header      Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/img/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/images/$1;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/spns/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/adverts/$1;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/avatars/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/avatars/$1;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/badges/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/badges/$1;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location ~ ^/tags/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/tags/$1;
+		proxy_hide_header Content-Type;
+		add_header Content-Type $custom_content_type;
 	}
 
 	location / {
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 2f4236de..9d80a070 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -8,6 +8,7 @@ defmodule Philomena.Images.Thumbnailer do
   alias Philomena.Images.Image
   alias Philomena.Processors
   alias Philomena.Analyzers
+  alias Philomena.Uploader
   alias Philomena.Sha512
   alias Philomena.Repo
   alias ExAws.S3
@@ -131,10 +132,7 @@ defmodule Philomena.Images.Thumbnailer do
   def upload_file(image, file, version_name) do
     path = Path.join(image_thumb_prefix(image), version_name)
 
-    file
-    |> S3.Upload.stream_file()
-    |> S3.upload(bucket(), path, @acl)
-    |> ExAws.request!()
+    Uploader.persist_file(path, file)
   end
 
   defp bulk_rename(file_names, source_prefix, target_prefix) do
diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex
index effb31a3..1b02a4b6 100644
--- a/lib/philomena/uploader.ex
+++ b/lib/philomena/uploader.ex
@@ -6,6 +6,7 @@ defmodule Philomena.Uploader do
   alias Philomena.Filename
   alias Philomena.Analyzers
   alias Philomena.Sha512
+  alias Philomena.Mime
   alias ExAws.S3
   import Ecto.Changeset
 
@@ -64,9 +65,19 @@ defmodule Philomena.Uploader do
     dest = Map.get(model, field(field_name))
     target = Path.join(file_root, dest)
 
-    source
+    persist_file(target, source)
+  end
+
+  @doc """
+  Persist an arbitrary file to storage at the given path with the correct
+  content type and permissions.
+  """
+  def persist_file(path, file) do
+    {_, mime} = Mime.file(path)
+
+    file
     |> S3.Upload.stream_file()
-    |> S3.upload(bucket(), target, acl: :public_read)
+    |> S3.upload(bucket(), path, acl: :public_read, content_type: mime)
     |> ExAws.request!()
   end
 

From 844d60279f63682fe52bdedaa21c276b9bb7b5de Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 10 Feb 2022 19:17:53 -0500
Subject: [PATCH 13/39] Avoid naming conflicts in development

---
 docker-compose.yml    | 4 ++--
 docker/web/nginx.conf | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 88443e52..dd5fe631 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -25,8 +25,8 @@ services:
       - AVATAR_URL_ROOT=/avatars
       - ADVERT_URL_ROOT=/spns
       - IMAGE_URL_ROOT=/img
-      - BADGE_URL_ROOT=/badges
-      - TAG_URL_ROOT=/tags
+      - BADGE_URL_ROOT=/badge-img
+      - TAG_URL_ROOT=/tag-img
       - ELASTICSEARCH_URL=http://elasticsearch:9200
       - REDIS_HOST=redis
       - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev
diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index 969cb6e1..59b80bc7 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -66,7 +66,7 @@ server {
 		add_header Content-Type $custom_content_type;
 	}
 
-	location ~ ^/badges/(.+) {
+	location ~ ^/badge-img/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/badges/$1;
@@ -74,7 +74,7 @@ server {
 		add_header Content-Type $custom_content_type;
 	}
 
-	location ~ ^/tags/(.+) {
+	location ~ ^/tag-img/(.+) {
 		expires max;
 		add_header Cache-Control public;
 		proxy_pass http://s3/$BUCKET_NAME/tags/$1;

From d5f8bbc656119fa9ed8ca40dd29b7b150c082915 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 10 Feb 2022 19:18:47 -0500
Subject: [PATCH 14/39] Create symlink target too

---
 docker/app/run-development | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/app/run-development b/docker/app/run-development
index 90a315d6..4e03fab9 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -1,6 +1,7 @@
 #!/usr/bin/env sh
 
 # Create S3 dirs
+mkdir -p /srv/philomena/priv/static/system/images/thumbs
 mkdir -p /srv/philomena/priv/s3/philomena
 ln -sf /srv/philomena/priv/static/system/images/thumbs /srv/philomena/priv/s3/philomena/images
 ln -sf /srv/philomena/priv/static/system/images /srv/philomena/priv/s3/philomena/adverts

From c9b1d3fc6312a75b0637c462c21bc0f3ef3201f4 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 10 Feb 2022 20:02:38 -0500
Subject: [PATCH 15/39] Don't unpersist anymore

---
 docker/app/run-development       | 5 +++++
 lib/philomena/images.ex          | 3 ---
 lib/philomena/images/uploader.ex | 4 ----
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/docker/app/run-development b/docker/app/run-development
index 4e03fab9..bc209c9a 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -43,6 +43,11 @@ until wget -qO - elasticsearch:9200; do
   sleep 2
 done
 
+until wget -qO - files/philomena; do
+  echo -n "."
+  sleep 2
+done
+
 echo
 
 # Try to create the database if it doesn't exist yet
diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex
index a244f280..7ecaf8ff 100644
--- a/lib/philomena/images.ex
+++ b/lib/philomena/images.ex
@@ -109,7 +109,6 @@ defmodule Philomena.Images do
     |> case do
       {:ok, %{image: image}} = result ->
         Uploader.persist_upload(image)
-        Uploader.unpersist_old_upload(image)
 
         repair_image(image)
         reindex_image(image)
@@ -195,7 +194,6 @@ defmodule Philomena.Images do
     |> Repo.update()
     |> case do
       {:ok, image} ->
-        Uploader.unpersist_old_upload(image)
         purge_files(image, image.hidden_image_key)
         Thumbnailer.destroy_thumbnails(image)
 
@@ -262,7 +260,6 @@ defmodule Philomena.Images do
     |> case do
       {:ok, image} ->
         Uploader.persist_upload(image)
-        Uploader.unpersist_old_upload(image)
 
         repair_image(image)
         purge_files(image, image.hidden_image_key)
diff --git a/lib/philomena/images/uploader.ex b/lib/philomena/images/uploader.ex
index 8419b805..39111e11 100644
--- a/lib/philomena/images/uploader.ex
+++ b/lib/philomena/images/uploader.ex
@@ -14,8 +14,4 @@ defmodule Philomena.Images.Uploader do
   def persist_upload(image) do
     Thumbnailer.upload_file(image, image.uploaded_image, "full.#{image.image_format}")
   end
-
-  def unpersist_old_upload(image) do
-    Thumbnailer.destroy_thumbnails(image)
-  end
 end

From 3ee8179cc8d0b67e752cf11b2162bfeed461026a Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 12 Feb 2022 13:44:42 -0500
Subject: [PATCH 16/39] Add uploader task

---
 docker/app/run-development          |   2 +-
 lib/mix/tasks/upload_to_s3.ex       | 115 ++++++++++++++++++++++++++++
 lib/philomena/images/thumbnailer.ex |   6 +-
 3 files changed, 119 insertions(+), 4 deletions(-)
 create mode 100644 lib/mix/tasks/upload_to_s3.ex

diff --git a/docker/app/run-development b/docker/app/run-development
index bc209c9a..9486d334 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -43,7 +43,7 @@ until wget -qO - elasticsearch:9200; do
   sleep 2
 done
 
-until wget -qO - files/philomena; do
+until wget -qO /dev/null files/philomena; do
   echo -n "."
   sleep 2
 done
diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
new file mode 100644
index 00000000..166be286
--- /dev/null
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -0,0 +1,115 @@
+defmodule Mix.Tasks.UploadToS3 do
+  use Mix.Task
+
+  alias Philomena.{
+    Adverts.Advert,
+    Badges.Badge,
+    Images.Image,
+    Tags.Tag,
+    Users.User
+  }
+
+  alias Philomena.Images.Thumbnailer
+  alias Philomena.Mime
+  alias Philomena.Batch
+  alias ExAws.S3
+  import Ecto.Query
+
+  @shortdoc "Dumps existing image files to S3 storage backend"
+  @requirements ["app.start"]
+  @impl Mix.Task
+  def run([concurrency | args]) do
+    {concurrency, _} = Integer.parse(concurrency)
+
+    if Enum.member?(args, "--adverts") do
+      file_root = Application.fetch_env!(:philomena, :advert_file_root)
+      new_file_root = System.get_env("NEW_ADVERT_FILE_ROOT", "adverts")
+
+      IO.puts "\nAdverts:"
+      upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--avatars") do
+      file_root = Application.fetch_env!(:philomena, :avatar_file_root)
+      new_file_root = System.get_env("NEW_AVATAR_FILE_ROOT", "avatars")
+
+      IO.puts "\nAvatars:"
+      upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, "avatar")
+    end
+
+    if Enum.member?(args, "--badges") do
+      file_root = Application.fetch_env!(:philomena, :badge_file_root)
+      new_file_root = System.get_env("NEW_BADGE_FILE_ROOT", "badges")
+
+      IO.puts "\nBadges:"
+      upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--tags") do
+      file_root = Application.fetch_env!(:philomena, :tag_file_root)
+      new_file_root = System.get_env("NEW_TAG_FILE_ROOT", "tags")
+
+      IO.puts "\nTags:"
+      upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, "image")
+    end
+
+    if Enum.member?(args, "--images") do
+      # Temporarily adjust the file root so that the thumbs are picked up
+      file_root = Application.fetch_env!(:philomena, :image_file_root) <> "thumbs"
+      Application.put_env(:philomena, :image_file_root, file_root)
+
+      new_file_root = System.get_env("NEW_IMAGE_FILE_ROOT", "images")
+
+      IO.puts "\nImages:"
+      upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
+    end
+  end
+
+  defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
+    Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+      Task.async_stream(models, &upload_typical_model(&1, file_root, new_file_root, field_name))
+
+      IO.write "\r#{hd(models).id}"
+    end)
+  end
+
+  defp upload_typical_model(model, file_root, new_file_root, field_name) do
+    path = Path.join(file_root, Map.fetch!(model, field_name))
+
+    if File.exists?(path) do
+      put_file(path, Path.join(new_file_root, field_name))
+    end
+  end
+
+  defp upload_images(queryable, batch_size, file_root, new_file_root) do
+    Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
+      Task.async_stream(models, &upload_image_model(&1, file_root, new_file_root))
+
+      IO.write "\r#{hd(models).id}"
+    end)
+  end
+
+  defp upload_image_model(model, file_root, new_file_root) do
+    path_prefix = Thumbnailer.image_thumb_prefix(model)
+
+    Thumbnailer.all_versions(model)
+    |> Enum.map(fn version ->
+      path = Path.join([file_root, path_prefix, version])
+      new_path = Path.join([new_file_root, path_prefix, version])
+
+      put_file(path, new_path)
+    end)
+  end
+
+  defp put_file(path, uploaded_path) do
+    mime = Mime.file(path)
+    contents = File.read!(path)
+
+    S3.put_object(bucket(), uploaded_path, contents, acl: :public_read, content_type: mime)
+    |> ExAws.request!()
+  end
+
+  defp bucket do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
+end
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 9d80a070..a093612d 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -153,7 +153,7 @@ defmodule Philomena.Images.Thumbnailer do
     end)
   end
 
-  defp all_versions(image) do
+  def all_versions(image) do
     generated = Processors.versions(image.image_mime_type, generated_sizes(image))
     full = ["full.#{image.image_format}"]
 
@@ -163,10 +163,10 @@ defmodule Philomena.Images.Thumbnailer do
   # This method wraps the following two for code that doesn't care
   # and just wants the files (most code should take this path)
 
-  defp image_thumb_prefix(%{hidden_from_users: true} = image),
+  def image_thumb_prefix(%{hidden_from_users: true} = image),
     do: hidden_image_thumb_prefix(image, image.hidden_image_key)
 
-  defp image_thumb_prefix(image),
+  def image_thumb_prefix(image),
     do: visible_image_thumb_prefix(image)
 
   # These methods handle the actual distinction between the two

From 4e7b9016bc747b34d1c3038bc55d00b589169a76 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 13 Feb 2022 11:50:19 -0500
Subject: [PATCH 17/39] Change file query to HEAD

---
 docker/app/run-development | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/app/run-development b/docker/app/run-development
index 9486d334..3fce8e72 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -43,7 +43,7 @@ until wget -qO - elasticsearch:9200; do
   sleep 2
 done
 
-until wget -qO /dev/null files/philomena; do
+until wget -qO /dev/null --spider files/philomena; do
   echo -n "."
   sleep 2
 done

From d67fd7ebb774f065d62dcffd6d7b87f7e0d77f30 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 13 Feb 2022 11:50:35 -0500
Subject: [PATCH 18/39] Add dev shim to upload task for local testing

---
 lib/mix/tasks/upload_to_s3.ex | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 166be286..cb250c6b 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -21,6 +21,14 @@ defmodule Mix.Tasks.UploadToS3 do
   def run([concurrency | args]) do
     {concurrency, _} = Integer.parse(concurrency)
 
+    if Enum.member?(args, "--dev") do
+      Application.put_env(:philomena, :advert_file_root, "priv/static/system/images/adverts")
+      Application.put_env(:philomena, :avatar_file_root, "priv/static/system/images/avatars")
+      Application.put_env(:philomena, :badge_file_root, "priv/static/system/images")
+      Application.put_env(:philomena, :image_file_root, "priv/static/system/images")
+      Application.put_env(:philomena, :tag_file_root, "priv/static/system/images")
+    end
+
     if Enum.member?(args, "--adverts") do
       file_root = Application.fetch_env!(:philomena, :advert_file_root)
       new_file_root = System.get_env("NEW_ADVERT_FILE_ROOT", "adverts")

From 06d557495a0f8ec80f27e66e50d5b27c778052ec Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 13 Feb 2022 12:13:28 -0500
Subject: [PATCH 19/39] Do prefixes the correct way around

---
 docker/app/run-development    |  5 ---
 lib/mix/tasks/upload_to_s3.ex | 61 +++++++++++++++++------------------
 2 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/docker/app/run-development b/docker/app/run-development
index 3fce8e72..4e03fab9 100755
--- a/docker/app/run-development
+++ b/docker/app/run-development
@@ -43,11 +43,6 @@ until wget -qO - elasticsearch:9200; do
   sleep 2
 done
 
-until wget -qO /dev/null --spider files/philomena; do
-  echo -n "."
-  sleep 2
-done
-
 echo
 
 # Try to create the database if it doesn't exist yet
diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index cb250c6b..89eaa6a9 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -21,52 +21,44 @@ defmodule Mix.Tasks.UploadToS3 do
   def run([concurrency | args]) do
     {concurrency, _} = Integer.parse(concurrency)
 
-    if Enum.member?(args, "--dev") do
-      Application.put_env(:philomena, :advert_file_root, "priv/static/system/images/adverts")
-      Application.put_env(:philomena, :avatar_file_root, "priv/static/system/images/avatars")
-      Application.put_env(:philomena, :badge_file_root, "priv/static/system/images")
-      Application.put_env(:philomena, :image_file_root, "priv/static/system/images")
-      Application.put_env(:philomena, :tag_file_root, "priv/static/system/images")
-    end
-
     if Enum.member?(args, "--adverts") do
-      file_root = Application.fetch_env!(:philomena, :advert_file_root)
-      new_file_root = System.get_env("NEW_ADVERT_FILE_ROOT", "adverts")
+      file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts")
+      new_file_root = Application.fetch_env!(:philomena, :advert_file_root)
 
       IO.puts "\nAdverts:"
-      upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, "image")
+      upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, :image)
     end
 
     if Enum.member?(args, "--avatars") do
-      file_root = Application.fetch_env!(:philomena, :avatar_file_root)
-      new_file_root = System.get_env("NEW_AVATAR_FILE_ROOT", "avatars")
+      file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars")
+      new_file_root = Application.fetch_env!(:philomena, :avatar_file_root)
 
       IO.puts "\nAvatars:"
-      upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, "avatar")
+      upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, :avatar)
     end
 
     if Enum.member?(args, "--badges") do
-      file_root = Application.fetch_env!(:philomena, :badge_file_root)
-      new_file_root = System.get_env("NEW_BADGE_FILE_ROOT", "badges")
+      file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images")
+      new_file_root = Application.fetch_env!(:philomena, :badge_file_root)
 
       IO.puts "\nBadges:"
-      upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, "image")
+      upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, :image)
     end
 
     if Enum.member?(args, "--tags") do
-      file_root = Application.fetch_env!(:philomena, :tag_file_root)
-      new_file_root = System.get_env("NEW_TAG_FILE_ROOT", "tags")
+      file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images")
+      new_file_root = Application.fetch_env!(:philomena, :tag_file_root)
 
       IO.puts "\nTags:"
-      upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, "image")
+      upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, :image)
     end
 
     if Enum.member?(args, "--images") do
-      # Temporarily adjust the file root so that the thumbs are picked up
-      file_root = Application.fetch_env!(:philomena, :image_file_root) <> "thumbs"
-      Application.put_env(:philomena, :image_file_root, file_root)
+      file_root = Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs")
+      new_file_root = Application.fetch_env!(:philomena, :image_file_root)
 
-      new_file_root = System.get_env("NEW_IMAGE_FILE_ROOT", "images")
+      # Temporarily set file root to empty path so we can get the proper prefix
+      Application.put_env(:philomena, :image_file_root, "")
 
       IO.puts "\nImages:"
       upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
@@ -75,23 +67,28 @@ defmodule Mix.Tasks.UploadToS3 do
 
   defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
     Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
-      Task.async_stream(models, &upload_typical_model(&1, file_root, new_file_root, field_name))
+      models
+      |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name))
+      |> Stream.run()
 
       IO.write "\r#{hd(models).id}"
     end)
   end
 
   defp upload_typical_model(model, file_root, new_file_root, field_name) do
-    path = Path.join(file_root, Map.fetch!(model, field_name))
+    field = Map.fetch!(model, field_name)
+    path = Path.join(file_root, field)
 
-    if File.exists?(path) do
-      put_file(path, Path.join(new_file_root, field_name))
+    if File.regular?(path) do
+      put_file(path, Path.join(new_file_root, field))
     end
   end
 
   defp upload_images(queryable, batch_size, file_root, new_file_root) do
     Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
-      Task.async_stream(models, &upload_image_model(&1, file_root, new_file_root))
+      models
+      |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root))
+      |> Stream.run()
 
       IO.write "\r#{hd(models).id}"
     end)
@@ -105,12 +102,14 @@ defmodule Mix.Tasks.UploadToS3 do
       path = Path.join([file_root, path_prefix, version])
       new_path = Path.join([new_file_root, path_prefix, version])
 
-      put_file(path, new_path)
+      if File.regular?(path) do
+        put_file(path, new_path)
+      end
     end)
   end
 
   defp put_file(path, uploaded_path) do
-    mime = Mime.file(path)
+    {_, mime} = Mime.file(path)
     contents = File.read!(path)
 
     S3.put_object(bucket(), uploaded_path, contents, acl: :public_read, content_type: mime)

From 3d756804b0e702d1104311f82f6956202c30f535 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 13 Feb 2022 12:32:35 -0500
Subject: [PATCH 20/39] Format

---
 lib/mix/tasks/upload_to_s3.ex | 54 +++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 89eaa6a9..599ea331 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -25,42 +25,72 @@ defmodule Mix.Tasks.UploadToS3 do
       file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts")
       new_file_root = Application.fetch_env!(:philomena, :advert_file_root)
 
-      IO.puts "\nAdverts:"
-      upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, :image)
+      IO.puts("\nAdverts:")
+
+      upload_typical(
+        where(Advert, [a], not is_nil(a.image)),
+        concurrency,
+        file_root,
+        new_file_root,
+        :image
+      )
     end
 
     if Enum.member?(args, "--avatars") do
       file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars")
       new_file_root = Application.fetch_env!(:philomena, :avatar_file_root)
 
-      IO.puts "\nAvatars:"
-      upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, :avatar)
+      IO.puts("\nAvatars:")
+
+      upload_typical(
+        where(User, [u], not is_nil(u.avatar)),
+        concurrency,
+        file_root,
+        new_file_root,
+        :avatar
+      )
     end
 
     if Enum.member?(args, "--badges") do
       file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images")
       new_file_root = Application.fetch_env!(:philomena, :badge_file_root)
 
-      IO.puts "\nBadges:"
-      upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, :image)
+      IO.puts("\nBadges:")
+
+      upload_typical(
+        where(Badge, [b], not is_nil(b.image)),
+        concurrency,
+        file_root,
+        new_file_root,
+        :image
+      )
     end
 
     if Enum.member?(args, "--tags") do
       file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images")
       new_file_root = Application.fetch_env!(:philomena, :tag_file_root)
 
-      IO.puts "\nTags:"
-      upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, :image)
+      IO.puts("\nTags:")
+
+      upload_typical(
+        where(Tag, [t], not is_nil(t.image)),
+        concurrency,
+        file_root,
+        new_file_root,
+        :image
+      )
     end
 
     if Enum.member?(args, "--images") do
-      file_root = Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs")
+      file_root =
+        Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs")
+
       new_file_root = Application.fetch_env!(:philomena, :image_file_root)
 
       # Temporarily set file root to empty path so we can get the proper prefix
       Application.put_env(:philomena, :image_file_root, "")
 
-      IO.puts "\nImages:"
+      IO.puts("\nImages:")
       upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
     end
   end
@@ -71,7 +101,7 @@ defmodule Mix.Tasks.UploadToS3 do
       |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name))
       |> Stream.run()
 
-      IO.write "\r#{hd(models).id}"
+      IO.write("\r#{hd(models).id}")
     end)
   end
 
@@ -90,7 +120,7 @@ defmodule Mix.Tasks.UploadToS3 do
       |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root))
       |> Stream.run()
 
-      IO.write "\r#{hd(models).id}"
+      IO.write("\r#{hd(models).id}")
     end)
   end
 

From 69ffce8a79947bf9a2cccd20f082831c12f9961a Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 13:46:23 -0400
Subject: [PATCH 21/39] Authenticate requests to remote S3 endpoint

---
 docker-compose.yml           |   9 ++
 docker/web/Dockerfile        |  18 ++--
 docker/web/aws-signature.lua | 149 +++++++++++++++++++++++++++++++
 docker/web/nginx.conf        | 167 ++++++++++++++++++++++++-----------
 4 files changed, 285 insertions(+), 58 deletions(-)
 create mode 100644 docker/web/aws-signature.lua

diff --git a/docker-compose.yml b/docker-compose.yml
index dd5fe631..9c87c458 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -87,8 +87,17 @@ services:
     build:
       context: .
       dockerfile: ./docker/web/Dockerfile
+      args:
+        - APP_DIR=/srv/philomena
+        - S3_SCHEME=http
+        - S3_HOST=files
+        - S3_PORT=80
+        - S3_BUCKET=philomena
     volumes:
       - .:/srv/philomena
+    environment:
+      - AWS_ACCESS_KEY_ID=local-identity
+      - AWS_SECRET_ACCESS_KEY=local-credential
     logging:
       driver: "none"
     depends_on:
diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile
index e76dab63..cf267ee0 100644
--- a/docker/web/Dockerfile
+++ b/docker/web/Dockerfile
@@ -1,7 +1,15 @@
-FROM nginx:1.21.4-alpine
-ENV APP_DIR /srv/philomena
-ENV BUCKET_NAME philomena
+FROM openresty/openresty:1.19.9.1-12-alpine
+ARG APP_DIR
+ARG S3_SCHEME
+ARG S3_HOST
+ARG S3_PORT
+ARG S3_BUCKET
+
+RUN apk add --no-cache gettext curl perl && opm get jkeys089/lua-resty-hmac=0.06 && mkdir -p /etc/nginx/lua
+COPY docker/web/aws-signature.lua /etc/nginx/lua
 COPY docker/web/nginx.conf /tmp/docker.nginx
-RUN envsubst '$APP_DIR $BUCKET_NAME' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf
+RUN envsubst '$APP_DIR $S3_SCHEME $S3_HOST $S3_PORT $S3_BUCKET' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf && \
+    echo 'env AWS_ACCESS_KEY_ID;' >> /usr/local/openresty/nginx/conf/nginx.conf && \
+    echo 'env AWS_SECRET_ACCESS_KEY;' >> /usr/local/openresty/nginx/conf/nginx.conf
 EXPOSE 80
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["openresty", "-g", "daemon off;"]
diff --git a/docker/web/aws-signature.lua b/docker/web/aws-signature.lua
new file mode 100644
index 00000000..fae28992
--- /dev/null
+++ b/docker/web/aws-signature.lua
@@ -0,0 +1,149 @@
+--[[
+Copyright 2018 JobTeaser
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+--]]
+
+local cjson = require('cjson')
+local resty_hmac = require('resty.hmac')
+local resty_sha256 = require('resty.sha256')
+local str = require('resty.string')
+
+local _M = { _VERSION = '0.1.2' }
+
+local function get_credentials ()
+  local access_key = os.getenv('AWS_ACCESS_KEY_ID')
+  local secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+
+  return {
+    access_key = access_key,
+    secret_key = secret_key
+  }
+end
+
+local function get_iso8601_basic(timestamp)
+  return os.date('!%Y%m%dT%H%M%SZ', timestamp)
+end
+
+local function get_iso8601_basic_short(timestamp)
+  return os.date('!%Y%m%d', timestamp)
+end
+
+local function get_derived_signing_key(keys, timestamp, region, service)
+  local h_date = resty_hmac:new('AWS4' .. keys['secret_key'], resty_hmac.ALGOS.SHA256)
+  h_date:update(get_iso8601_basic_short(timestamp))
+  k_date = h_date:final()
+
+  local h_region = resty_hmac:new(k_date, resty_hmac.ALGOS.SHA256)
+  h_region:update(region)
+  k_region = h_region:final()
+
+  local h_service = resty_hmac:new(k_region, resty_hmac.ALGOS.SHA256)
+  h_service:update(service)
+  k_service = h_service:final()
+
+  local h = resty_hmac:new(k_service, resty_hmac.ALGOS.SHA256)
+  h:update('aws4_request')
+  return h:final()
+end
+
+local function get_cred_scope(timestamp, region, service)
+  return get_iso8601_basic_short(timestamp)
+    .. '/' .. region
+    .. '/' .. service
+    .. '/aws4_request'
+end
+
+local function get_signed_headers()
+  return 'host;x-amz-content-sha256;x-amz-date'
+end
+
+local function get_sha256_digest(s)
+  local h = resty_sha256:new()
+  h:update(s or '')
+  return str.to_hex(h:final())
+end
+
+local function get_hashed_canonical_request(timestamp, host, uri)
+  local digest = get_sha256_digest(ngx.var.request_body)
+  local canonical_request = ngx.var.request_method .. '\n'
+    .. uri .. '\n'
+    .. '\n'
+    .. 'host:' .. host .. '\n'
+    .. 'x-amz-content-sha256:' .. digest .. '\n'
+    .. 'x-amz-date:' .. get_iso8601_basic(timestamp) .. '\n'
+    .. '\n'
+    .. get_signed_headers() .. '\n'
+    .. digest
+  return get_sha256_digest(canonical_request)
+end
+
+local function get_string_to_sign(timestamp, region, service, host, uri)
+  return 'AWS4-HMAC-SHA256\n'
+    .. get_iso8601_basic(timestamp) .. '\n'
+    .. get_cred_scope(timestamp, region, service) .. '\n'
+    .. get_hashed_canonical_request(timestamp, host, uri)
+end
+
+local function get_signature(derived_signing_key, string_to_sign)
+  local h = resty_hmac:new(derived_signing_key, resty_hmac.ALGOS.SHA256)
+  h:update(string_to_sign)
+  return h:final(nil, true)
+end
+
+local function get_authorization(keys, timestamp, region, service, host, uri)
+  local derived_signing_key = get_derived_signing_key(keys, timestamp, region, service)
+  local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri)
+  local auth = 'AWS4-HMAC-SHA256 '
+    .. 'Credential=' .. keys['access_key'] .. '/' .. get_cred_scope(timestamp, region, service)
+    .. ', SignedHeaders=' .. get_signed_headers()
+    .. ', Signature=' .. get_signature(derived_signing_key, string_to_sign)
+  return auth
+end
+
+local function get_service_and_region(host)
+  local patterns = {
+    {'s3.amazonaws.com', 's3', 'us-east-1'},
+    {'s3-external-1.amazonaws.com', 's3', 'us-east-1'},
+    {'s3%-([a-z0-9-]+)%.amazonaws%.com', 's3', nil}
+  }
+
+  for i,data in ipairs(patterns) do
+    local region = host:match(data[1])
+    if region ~= nil and data[3] == nil then
+      return data[2], region
+    elseif region ~= nil then
+      return data[2], data[3]
+    end
+  end
+
+  return 's3', 'auto'
+end
+
+function _M.aws_set_headers(host, uri)
+  local creds = get_credentials()
+  local timestamp = tonumber(ngx.time())
+  local service, region = get_service_and_region(host)
+  local auth = get_authorization(creds, timestamp, region, service, host, uri)
+
+  ngx.req.set_header('Authorization', auth)
+  ngx.req.set_header('Host', host)
+  ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp))
+end
+
+function _M.s3_set_headers(host, uri)
+  _M.aws_set_headers(host, uri)
+  ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body))
+end
+
+return _M
diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index 59b80bc7..788c9ee5 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -2,18 +2,47 @@ upstream philomena {
 	server app:4000 fail_timeout=0;
 }
 
-upstream s3 {
-	server files:80 fail_timeout=0;
+map $uri $custom_content_type {
+	default			"text/html";
+	~(.*\.png)$		"image/png";
+	~(.*\.jpe?g)$	"image/jpeg";
+	~(.*\.gif)$		"image/gif";
+	~(.*\.svg)$		"image/svg+xml";
+	~(.*\.mp4)$		"video/mp4";
+	~(.*\.webm)$	"video/webm";
 }
 
-map $uri $custom_content_type {
-	default		"text/html";
-	~(.*\.png)$	"image/png";
-	~(.*\.jpe?g)$	"image/jpeg";
-	~(.*\.gif)$	"image/gif";
-	~(.*\.svg)$	"image/svg+xml";
-	~(.*\.mp4)$	"video/mp4";
-	~(.*\.webm)$	"video/webm";
+lua_package_path '/etc/nginx/lua/?.lua;;';
+resolver 1.1.1.1 ipv6=off;
+
+init_by_lua_block {
+	aws_sig = require('aws-signature')
+
+	function clear_request()
+		-- Get rid of any client state that could cause
+		-- issues for the proxied request
+		for h, _ in pairs(ngx.req.get_headers()) do
+			if string.lower(h) ~= 'range' then
+				ngx.req.clear_header(h)
+			end
+		end
+
+		ngx.req.set_uri_args({})
+		ngx.req.discard_body()
+	end
+
+	function sign_aws_request()
+		-- The API token used should not allow writing, but
+		-- sanitize this anyway to stop an upstream error
+		if ngx.req.get_method() ~= 'GET' then
+			ngx.status = ngx.HTTP_UNAUTHORIZED
+			ngx.say('Unauthorized')
+			return ngx.exit(ngx.HTTP_UNAUTHORIZED)
+		end
+
+		clear_request()
+		aws_sig.s3_set_headers("$S3_HOST", ngx.var.uri)
+	end
 }
 
 server {
@@ -25,61 +54,93 @@ server {
 	client_max_body_size       125000000;
 	client_body_buffer_size    128k;
 
-	location ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
-		expires max;
-		add_header      Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
+		rewrite ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires 	max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
-		add_header Content-Disposition "attachment";
-		expires max;
-		add_header      Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
+		rewrite ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
+		add_header	Content-Disposition "attachment";
 	}
 
-	location ~ ^/img/(.+) {
-		expires max;
-		add_header Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/images/$1;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/img/(.+)$ {
+		rewrite ^/img/(.+)$ "/$S3_BUCKET/images/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/spns/(.+) {
-		expires max;
-		add_header Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/adverts/$1;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/spns/(.+) {
+		rewrite ^/spns/(.+)$ "/$S3_BUCKET/adverts/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/avatars/(.+) {
-		expires max;
-		add_header Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/avatars/$1;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/avatars/(.+) {
+		rewrite ^/avatars/(.+)$ "/$S3_BUCKET/avatars/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/badge-img/(.+) {
-		expires max;
-		add_header Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/badges/$1;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	# The following two location blocks use an -img suffix to avoid
+	# conflicting with the application routes. In production, this
+	# is not necessary since assets will be on a distinct domain.
+
+	location  ~ ^/badge-img/(.+) {
+		rewrite ^/badge-img/(.+)$ "/$S3_BUCKET/badges/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/tag-img/(.+) {
-		expires max;
-		add_header Cache-Control public;
-		proxy_pass http://s3/$BUCKET_NAME/tags/$1;
-		proxy_hide_header Content-Type;
-		add_header Content-Type $custom_content_type;
+	location  ~ ^/tag-img/(.+) {
+		rewrite ^/tag-img/(.+)$ "/$S3_BUCKET/tags/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
 	location / {

From 96372ec921e49064b233a2a5e8c53583613ecf1a Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 13:54:45 -0400
Subject: [PATCH 22/39] Use a private bucket

---
 lib/mix/tasks/upload_to_s3.ex       | 2 +-
 lib/philomena/images/thumbnailer.ex | 4 +---
 lib/philomena/uploader.ex           | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 599ea331..bcdd8148 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -142,7 +142,7 @@ defmodule Mix.Tasks.UploadToS3 do
     {_, mime} = Mime.file(path)
     contents = File.read!(path)
 
-    S3.put_object(bucket(), uploaded_path, contents, acl: :public_read, content_type: mime)
+    S3.put_object(bucket(), uploaded_path, contents, content_type: mime)
     |> ExAws.request!()
   end
 
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index a093612d..078ed0bb 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -23,8 +23,6 @@ defmodule Philomena.Images.Thumbnailer do
     tall: {1024, 4096}
   ]
 
-  @acl [acl: :public_read]
-
   def thumbnail_versions do
     @versions
   end
@@ -140,7 +138,7 @@ defmodule Philomena.Images.Thumbnailer do
       source = Path.join(source_prefix, name)
       target = Path.join(target_prefix, name)
 
-      ExAws.request(S3.put_object_copy(bucket(), target, bucket(), source, @acl))
+      ExAws.request(S3.put_object_copy(bucket(), target, bucket(), source))
       ExAws.request(S3.delete_object(bucket(), source))
     end)
   end
diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex
index 1b02a4b6..9b35ebbb 100644
--- a/lib/philomena/uploader.ex
+++ b/lib/philomena/uploader.ex
@@ -73,11 +73,11 @@ defmodule Philomena.Uploader do
   content type and permissions.
   """
   def persist_file(path, file) do
-    {_, mime} = Mime.file(path)
+    {_, mime} = Mime.file(file)
 
     file
     |> S3.Upload.stream_file()
-    |> S3.upload(bucket(), path, acl: :public_read, content_type: mime)
+    |> S3.upload(bucket(), path, content_type: mime)
     |> ExAws.request!()
   end
 

From 76bf7f292a3e1819aed47eab0944902b8889196c Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 17:22:29 -0400
Subject: [PATCH 23/39] Add live replication support

---
 config/runtime.exs                  |  22 ++++--
 lib/mix/tasks/upload_to_s3.ex       |  13 +---
 lib/philomena/images/thumbnailer.ex |  19 ++---
 lib/philomena/objects.ex            | 104 ++++++++++++++++++++++++++++
 lib/philomena/uploader.ex           |  18 +----
 5 files changed, 134 insertions(+), 42 deletions(-)
 create mode 100644 lib/philomena/objects.ex

diff --git a/config/runtime.exs b/config/runtime.exs
index 46a143b8..43cd85ea 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -31,7 +31,6 @@ config :philomena,
   tag_url_root: System.fetch_env!("TAG_URL_ROOT"),
   redis_host: System.get_env("REDIS_HOST", "localhost"),
   proxy_host: System.get_env("PROXY_HOST"),
-  s3_bucket: System.fetch_env!("S3_BUCKET"),
   camo_host: System.get_env("CAMO_HOST"),
   camo_key: System.get_env("CAMO_KEY"),
   cdn_host: System.fetch_env!("CDN_HOST")
@@ -67,11 +66,26 @@ if is_nil(System.get_env("START_WORKER")) do
   config :exq, queues: []
 end
 
-# S3 config
-config :ex_aws, :s3,
+# S3/Object store config
+config :philomena, :s3_primary_options,
+  region: System.get_env("S3_REGION", "us-east-1"),
   scheme: System.fetch_env!("S3_SCHEME"),
   host: System.fetch_env!("S3_HOST"),
-  port: System.fetch_env!("S3_PORT")
+  port: System.fetch_env!("S3_PORT"),
+  access_key_id: System.fetch_env!("AWS_ACCESS_KEY_ID"),
+  secret_access_key: System.fetch_env!("AWS_SECRET_ACCESS_KEY")
+
+config :philomena, :s3_primary_bucket, System.fetch_env!("S3_BUCKET")
+
+config :philomena, :s3_secondary_options,
+  region: System.get_env("ALT_S3_REGION", "us-east-1"),
+  scheme: System.get_env("ALT_S3_SCHEME"),
+  host: System.get_env("ALT_S3_HOST"),
+  port: System.get_env("ALT_S3_PORT"),
+  access_key_id: System.get_env("ALT_AWS_ACCESS_KEY_ID"),
+  secret_access_key: System.get_env("ALT_AWS_SECRET_ACCESS_KEY")
+
+config :philomena, :s3_secondary_bucket, System.get_env("ALT_S3_BUCKET")
 
 if config_env() != :test do
   # Database config
diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index bcdd8148..3cc5b6c5 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -10,9 +10,8 @@ defmodule Mix.Tasks.UploadToS3 do
   }
 
   alias Philomena.Images.Thumbnailer
-  alias Philomena.Mime
+  alias Philomena.Objects
   alias Philomena.Batch
-  alias ExAws.S3
   import Ecto.Query
 
   @shortdoc "Dumps existing image files to S3 storage backend"
@@ -139,14 +138,6 @@ defmodule Mix.Tasks.UploadToS3 do
   end
 
   defp put_file(path, uploaded_path) do
-    {_, mime} = Mime.file(path)
-    contents = File.read!(path)
-
-    S3.put_object(bucket(), uploaded_path, contents, content_type: mime)
-    |> ExAws.request!()
-  end
-
-  defp bucket do
-    Application.fetch_env!(:philomena, :s3_bucket)
+    Objects.put(uploaded_path, path)
   end
 end
diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index 078ed0bb..b0e6cd1d 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -9,9 +9,9 @@ defmodule Philomena.Images.Thumbnailer do
   alias Philomena.Processors
   alias Philomena.Analyzers
   alias Philomena.Uploader
+  alias Philomena.Objects
   alias Philomena.Sha512
   alias Philomena.Repo
-  alias ExAws.S3
 
   @versions [
     thumb_tiny: {50, 50},
@@ -122,7 +122,7 @@ defmodule Philomena.Images.Thumbnailer do
     tempfile = Briefly.create!(extname: ".#{image.image_format}")
     path = Path.join(image_thumb_prefix(image), "full.#{image.image_format}")
 
-    ExAws.request!(S3.download_file(bucket(), path, tempfile))
+    Objects.download_file(path, tempfile)
 
     tempfile
   end
@@ -138,17 +138,15 @@ defmodule Philomena.Images.Thumbnailer do
       source = Path.join(source_prefix, name)
       target = Path.join(target_prefix, name)
 
-      ExAws.request(S3.put_object_copy(bucket(), target, bucket(), source))
-      ExAws.request(S3.delete_object(bucket(), source))
+      Objects.copy(source, target)
+      Objects.delete(source)
     end)
   end
 
   defp bulk_delete(file_names, prefix) do
-    Enum.map(file_names, fn name ->
-      target = Path.join(prefix, name)
-
-      ExAws.request(S3.delete_object(bucket(), target))
-    end)
+    file_names
+    |> Enum.map(&Path.join(prefix, &1))
+    |> Objects.delete_multiple()
   end
 
   def all_versions(image) do
@@ -189,7 +187,4 @@ defmodule Philomena.Images.Thumbnailer do
 
   defp image_url_root,
     do: Application.fetch_env!(:philomena, :image_url_root)
-
-  defp bucket,
-    do: Application.fetch_env!(:philomena, :s3_bucket)
 end
diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
new file mode 100644
index 00000000..35d71fe4
--- /dev/null
+++ b/lib/philomena/objects.ex
@@ -0,0 +1,104 @@
+defmodule Philomena.Objects do
+  @moduledoc """
+  Replication wrapper for object storage backends.
+  """
+  alias Philomena.Mime
+
+  #
+  # Fetch a key from the primary storage backend and
+  # write it into the destination file.
+  #
+  @spec download_file(String.t(), String.t()) :: any()
+  def download_file(key, file_path) do
+    [opts] = primary_opts()
+
+    contents =
+      ExAws.S3.get_object(opts[:bucket], key)
+      |> ExAws.request!(opts[:config_overrides])
+
+    File.write!(file_path, contents.body)
+  end
+
+  #
+  # Upload a file using a single API call, writing the
+  # contents from the given path to storage.
+  #
+  @spec put(String.t(), String.t()) :: any()
+  def put(key, file_path) do
+    {_, mime} = Mime.file(file_path)
+    contents = File.read!(file_path)
+
+    run_all(fn opts ->
+      ExAws.S3.put_object(opts[:bucket], key, contents, content_type: mime)
+      |> ExAws.request!(opts[:config_overrides])
+    end)
+  end
+
+  #
+  # Copies a key from the source to the destination,
+  # overwriting the destination object if its exists.
+  #
+  @spec copy(String.t(), String.t()) :: any()
+  def copy(source_key, dest_key) do
+    run_all(fn opts ->
+      ExAws.S3.put_object_copy(opts[:bucket], dest_key, opts[:bucket], source_key)
+      |> ExAws.request!(opts[:config_overrides])
+    end)
+  end
+
+  #
+  # Removes the key from storage.
+  #
+  @spec delete(String.t()) :: any()
+  def delete(key) do
+    run_all(fn opts ->
+      ExAws.S3.delete_object(opts[:bucket], key)
+      |> ExAws.request!(opts[:config_overrides])
+    end)
+  end
+
+  #
+  # Removes all given keys from storage.
+  #
+  @spec delete_multiple([String.t()]) :: any()
+  def delete_multiple(keys) do
+    run_all(fn opts ->
+      ExAws.S3.delete_multiple_objects(opts[:bucket], keys)
+      |> ExAws.request!(opts[:config_overrides])
+    end)
+  end
+
+  defp run_all(fun) do
+    backends()
+    |> Task.async_stream(fun)
+    |> Stream.run()
+  end
+
+  defp backends do
+    primary_opts() ++ replica_opts()
+  end
+
+  defp primary_opts do
+    [
+      %{
+        config_overrides: Application.fetch_env!(:philomena, :s3_primary_options),
+        bucket: Application.fetch_env!(:philomena, :s3_primary_bucket)
+      }
+    ]
+  end
+
+  defp replica_opts do
+    replica_bucket = Application.get_env(:philomena, :s3_secondary_bucket)
+
+    if not is_nil(replica_bucket) do
+      [
+        %{
+          config_overrides: Application.fetch_env!(:philomena, :s3_secondary_options),
+          bucket: replica_bucket
+        }
+      ]
+    else
+      []
+    end
+  end
+end
diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex
index 9b35ebbb..76ba4c8d 100644
--- a/lib/philomena/uploader.ex
+++ b/lib/philomena/uploader.ex
@@ -5,9 +5,8 @@ defmodule Philomena.Uploader do
 
   alias Philomena.Filename
   alias Philomena.Analyzers
+  alias Philomena.Objects
   alias Philomena.Sha512
-  alias Philomena.Mime
-  alias ExAws.S3
   import Ecto.Changeset
 
   @doc """
@@ -73,12 +72,7 @@ defmodule Philomena.Uploader do
   content type and permissions.
   """
   def persist_file(path, file) do
-    {_, mime} = Mime.file(file)
-
-    file
-    |> S3.Upload.stream_file()
-    |> S3.upload(bucket(), path, content_type: mime)
-    |> ExAws.request!()
+    Objects.put(path, file)
   end
 
   @doc """
@@ -117,9 +111,7 @@ defmodule Philomena.Uploader do
   defp try_remove(nil, _file_root), do: nil
 
   defp try_remove(file, file_root) do
-    path = Path.join(file_root, file)
-
-    ExAws.request!(S3.delete_object(bucket(), path))
+    Objects.delete(Path.join(file_root, file))
   end
 
   defp prefix_attributes(map, prefix),
@@ -130,8 +122,4 @@ defmodule Philomena.Uploader do
   defp remove_key(field_name), do: "removed_#{field_name}"
 
   defp field(field_name), do: String.to_existing_atom(field_name)
-
-  defp bucket do
-    Application.fetch_env!(:philomena, :s3_bucket)
-  end
 end

From 07862b0d0c9e2df227504649299b5e314d52caed Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 20:03:33 -0400
Subject: [PATCH 24/39] Require a start date for upload_to_s3 task

---
 lib/mix/tasks/upload_to_s3.ex | 54 ++++++++++++++++++++++++++---------
 1 file changed, 40 insertions(+), 14 deletions(-)

diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 3cc5b6c5..02e20185 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -17,17 +17,37 @@ defmodule Mix.Tasks.UploadToS3 do
   @shortdoc "Dumps existing image files to S3 storage backend"
   @requirements ["app.start"]
   @impl Mix.Task
-  def run([concurrency | args]) do
-    {concurrency, _} = Integer.parse(concurrency)
+  def run(args) do
+    {args, rest} =
+      OptionParser.parse_head!(args,
+        strict: [
+          concurrency: :integer,
+          adverts: :boolean,
+          avatars: :boolean,
+          badges: :boolean,
+          tags: :boolean,
+          images: :boolean
+        ]
+      )
 
-    if Enum.member?(args, "--adverts") do
+    concurrency = Keyword.get(args, :concurrency, 4)
+
+    time =
+      with [time] <- rest,
+           {:ok, time, _} <- DateTime.from_iso8601(time) do
+        time
+      else
+        _ -> raise ArgumentError, "Must provide a RFC3339 start time, like 1970-01-01T00:00:00Z"
+      end
+
+    if args[:adverts] do
       file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts")
       new_file_root = Application.fetch_env!(:philomena, :advert_file_root)
 
       IO.puts("\nAdverts:")
 
       upload_typical(
-        where(Advert, [a], not is_nil(a.image)),
+        where(Advert, [a], not is_nil(a.image) and a.updated_at >= ^time),
         concurrency,
         file_root,
         new_file_root,
@@ -35,14 +55,14 @@ defmodule Mix.Tasks.UploadToS3 do
       )
     end
 
-    if Enum.member?(args, "--avatars") do
+    if args[:avatars] do
       file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars")
       new_file_root = Application.fetch_env!(:philomena, :avatar_file_root)
 
       IO.puts("\nAvatars:")
 
       upload_typical(
-        where(User, [u], not is_nil(u.avatar)),
+        where(User, [u], not is_nil(u.avatar) and u.updated_at >= ^time),
         concurrency,
         file_root,
         new_file_root,
@@ -50,14 +70,14 @@ defmodule Mix.Tasks.UploadToS3 do
       )
     end
 
-    if Enum.member?(args, "--badges") do
+    if args[:badges] do
       file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images")
       new_file_root = Application.fetch_env!(:philomena, :badge_file_root)
 
       IO.puts("\nBadges:")
 
       upload_typical(
-        where(Badge, [b], not is_nil(b.image)),
+        where(Badge, [b], not is_nil(b.image) and b.updated_at >= ^time),
         concurrency,
         file_root,
         new_file_root,
@@ -65,14 +85,14 @@ defmodule Mix.Tasks.UploadToS3 do
       )
     end
 
-    if Enum.member?(args, "--tags") do
+    if args[:tags] do
       file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images")
       new_file_root = Application.fetch_env!(:philomena, :tag_file_root)
 
       IO.puts("\nTags:")
 
       upload_typical(
-        where(Tag, [t], not is_nil(t.image)),
+        where(Tag, [t], not is_nil(t.image) and t.updated_at >= ^time),
         concurrency,
         file_root,
         new_file_root,
@@ -80,7 +100,7 @@ defmodule Mix.Tasks.UploadToS3 do
       )
     end
 
-    if Enum.member?(args, "--images") do
+    if args[:images] do
       file_root =
         Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs")
 
@@ -90,7 +110,13 @@ defmodule Mix.Tasks.UploadToS3 do
       Application.put_env(:philomena, :image_file_root, "")
 
       IO.puts("\nImages:")
-      upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
+
+      upload_images(
+        where(Image, [i], not is_nil(i.image) and i.updated_at >= ^time),
+        concurrency,
+        file_root,
+        new_file_root
+      )
     end
   end
 
@@ -100,7 +126,7 @@ defmodule Mix.Tasks.UploadToS3 do
       |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name))
       |> Stream.run()
 
-      IO.write("\r#{hd(models).id}")
+      IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
     end)
   end
 
@@ -119,7 +145,7 @@ defmodule Mix.Tasks.UploadToS3 do
       |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root))
       |> Stream.run()
 
-      IO.write("\r#{hd(models).id}")
+      IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
     end)
   end
 

From f2b6593da2e957a3c44d63131c84f71103e809bc Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 20:33:31 -0400
Subject: [PATCH 25/39] Add proxy cache

---
 docker/web/Dockerfile |  1 +
 docker/web/nginx.conf | 68 +++++++++++++------------------------------
 2 files changed, 21 insertions(+), 48 deletions(-)

diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile
index cf267ee0..f270a619 100644
--- a/docker/web/Dockerfile
+++ b/docker/web/Dockerfile
@@ -9,6 +9,7 @@ RUN apk add --no-cache gettext curl perl && opm get jkeys089/lua-resty-hmac=0.06
 COPY docker/web/aws-signature.lua /etc/nginx/lua
 COPY docker/web/nginx.conf /tmp/docker.nginx
 RUN envsubst '$APP_DIR $S3_SCHEME $S3_HOST $S3_PORT $S3_BUCKET' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf && \
+    mkdir -p /var/www/cache/tmp && \
     echo 'env AWS_ACCESS_KEY_ID;' >> /usr/local/openresty/nginx/conf/nginx.conf && \
     echo 'env AWS_SECRET_ACCESS_KEY;' >> /usr/local/openresty/nginx/conf/nginx.conf
 EXPOSE 80
diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index 788c9ee5..26dc7af3 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -45,6 +45,8 @@ init_by_lua_block {
 	end
 }
 
+proxy_cache_path /var/www/cache levels=1:2 keys_zone=s3-cache:8m max_size=1000m inactive=600m;
+
 server {
 	listen 80 default;
 	listen [::]:80;
@@ -54,10 +56,13 @@ server {
 	client_max_body_size       125000000;
 	client_body_buffer_size    128k;
 
-	location  ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
-		rewrite ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
+	location /$S3_BUCKET {
+		internal;
+
 		access_by_lua	 		"sign_aws_request()";
 		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_cache				s3-cache;
+		proxy_cache_valid		1h;
 		proxy_hide_header		Content-Type;
 		proxy_ssl_server_name	on;
 
@@ -68,51 +73,34 @@ server {
 
 	location  ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
 		rewrite ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
+
 		access_by_lua	 		"sign_aws_request()";
 		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_cache				s3-cache;
+		proxy_cache_valid		1h;
 		proxy_hide_header		Content-Type;
 		proxy_ssl_server_name	on;
 
-		expires		max;
+		expires 	max;
 		add_header	Cache-Control public;
 		add_header	Content-Type $custom_content_type;
 		add_header	Content-Disposition "attachment";
 	}
 
-	location  ~ ^/img/(.+)$ {
-		rewrite ^/img/(.+)$ "/$S3_BUCKET/images/$1" break;
-		access_by_lua	 		"sign_aws_request()";
-		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header		Content-Type;
-		proxy_ssl_server_name	on;
+	location  ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
+		rewrite ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" last;
+	}
 
-		expires		max;
-		add_header	Cache-Control public;
-		add_header	Content-Type $custom_content_type;
+	location  ~ ^/img/(.+)$ {
+		rewrite ^/img/(.+)$ "/$S3_BUCKET/images/$1" last;
 	}
 
 	location  ~ ^/spns/(.+) {
-		rewrite ^/spns/(.+)$ "/$S3_BUCKET/adverts/$1" break;
-		access_by_lua	 		"sign_aws_request()";
-		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header		Content-Type;
-		proxy_ssl_server_name	on;
-
-		expires		max;
-		add_header	Cache-Control public;
-		add_header	Content-Type $custom_content_type;
+		rewrite ^/spns/(.+)$ "/$S3_BUCKET/adverts/$1" last;
 	}
 
 	location  ~ ^/avatars/(.+) {
-		rewrite ^/avatars/(.+)$ "/$S3_BUCKET/avatars/$1" break;
-		access_by_lua	 		"sign_aws_request()";
-		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header		Content-Type;
-		proxy_ssl_server_name	on;
-
-		expires		max;
-		add_header	Cache-Control public;
-		add_header	Content-Type $custom_content_type;
+		rewrite ^/avatars/(.+)$ "/$S3_BUCKET/avatars/$1" last;
 	}
 
 	# The following two location blocks use an -img suffix to avoid
@@ -120,27 +108,11 @@ server {
 	# is not necessary since assets will be on a distinct domain.
 
 	location  ~ ^/badge-img/(.+) {
-		rewrite ^/badge-img/(.+)$ "/$S3_BUCKET/badges/$1" break;
-		access_by_lua	 		"sign_aws_request()";
-		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header		Content-Type;
-		proxy_ssl_server_name	on;
-
-		expires		max;
-		add_header	Cache-Control public;
-		add_header	Content-Type $custom_content_type;
+		rewrite ^/badge-img/(.+)$ "/$S3_BUCKET/badges/$1" last;
 	}
 
 	location  ~ ^/tag-img/(.+) {
-		rewrite ^/tag-img/(.+)$ "/$S3_BUCKET/tags/$1" break;
-		access_by_lua	 		"sign_aws_request()";
-		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header		Content-Type;
-		proxy_ssl_server_name	on;
-
-		expires		max;
-		add_header	Cache-Control public;
-		add_header	Content-Type $custom_content_type;
+		rewrite ^/tag-img/(.+)$ "/$S3_BUCKET/tags/$1" last;
 	}
 
 	location / {

From ce653a53ba3738de4e8f79e262d55cb502f9f328 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 14 May 2022 20:54:13 -0400
Subject: [PATCH 26/39] Silence sobelow warnings

---
 lib/philomena/objects.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 35d71fe4..6d05a6d8 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -8,6 +8,7 @@ defmodule Philomena.Objects do
   # Fetch a key from the primary storage backend and
   # write it into the destination file.
   #
+  # sobelow_skip ["Traversal.FileModule"]
   @spec download_file(String.t(), String.t()) :: any()
   def download_file(key, file_path) do
     [opts] = primary_opts()
@@ -23,6 +24,7 @@ defmodule Philomena.Objects do
   # Upload a file using a single API call, writing the
   # contents from the given path to storage.
   #
+  # sobelow_skip ["Traversal.FileModule"]
   @spec put(String.t(), String.t()) :: any()
   def put(key, file_path) do
     {_, mime} = Mime.file(file_path)

From ba0d231750da733853188035ec434b24521e78a0 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sun, 15 May 2022 09:44:17 -0400
Subject: [PATCH 27/39] Adjust timeouts

---
 lib/mix/tasks/upload_to_s3.ex | 6 ++++--
 lib/philomena/objects.ex      | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex
index 02e20185..0155514f 100644
--- a/lib/mix/tasks/upload_to_s3.ex
+++ b/lib/mix/tasks/upload_to_s3.ex
@@ -123,7 +123,9 @@ defmodule Mix.Tasks.UploadToS3 do
   defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do
     Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
       models
-      |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name))
+      |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name),
+        timeout: :infinity
+      )
       |> Stream.run()
 
       IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
@@ -142,7 +144,7 @@ defmodule Mix.Tasks.UploadToS3 do
   defp upload_images(queryable, batch_size, file_root, new_file_root) do
     Batch.record_batches(queryable, [batch_size: batch_size], fn models ->
       models
-      |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root))
+      |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root), timeout: :infinity)
       |> Stream.run()
 
       IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 6d05a6d8..2232617f 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -72,7 +72,7 @@ defmodule Philomena.Objects do
 
   defp run_all(fun) do
     backends()
-    |> Task.async_stream(fun)
+    |> Task.async_stream(fun, timeout: :infinity)
     |> Stream.run()
   end
 

From 18461027e4056a4c0b39838a758e3dda6c54c103 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 16 May 2022 17:16:54 -0400
Subject: [PATCH 28/39] More timeouts

---
 config/runtime.exs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 43cd85ea..2e211f30 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -73,7 +73,8 @@ config :philomena, :s3_primary_options,
   host: System.fetch_env!("S3_HOST"),
   port: System.fetch_env!("S3_PORT"),
   access_key_id: System.fetch_env!("AWS_ACCESS_KEY_ID"),
-  secret_access_key: System.fetch_env!("AWS_SECRET_ACCESS_KEY")
+  secret_access_key: System.fetch_env!("AWS_SECRET_ACCESS_KEY"),
+  http_opts: [timeout: 180_000, recv_timeout: 180_000]
 
 config :philomena, :s3_primary_bucket, System.fetch_env!("S3_BUCKET")
 
@@ -83,10 +84,17 @@ config :philomena, :s3_secondary_options,
   host: System.get_env("ALT_S3_HOST"),
   port: System.get_env("ALT_S3_PORT"),
   access_key_id: System.get_env("ALT_AWS_ACCESS_KEY_ID"),
-  secret_access_key: System.get_env("ALT_AWS_SECRET_ACCESS_KEY")
+  secret_access_key: System.get_env("ALT_AWS_SECRET_ACCESS_KEY"),
+  http_opts: [timeout: 180_000, recv_timeout: 180_000]
 
 config :philomena, :s3_secondary_bucket, System.get_env("ALT_S3_BUCKET")
 
+config :ex_aws, :hackney_opts,
+  timeout: 180_000,
+  recv_timeout: 180_000,
+  use_default_pool: false,
+  pool: false
+
 if config_env() != :test do
   # Database config
   config :philomena, Philomena.Repo,

From 167da3dcdbbe3a24d0a7be8eceb0a8d4687d8504 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 16 May 2022 17:18:29 -0400
Subject: [PATCH 29/39] Prioritize bucket URL matches

---
 docker/web/nginx.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf
index 26dc7af3..218fe896 100644
--- a/docker/web/nginx.conf
+++ b/docker/web/nginx.conf
@@ -56,7 +56,7 @@ server {
 	client_max_body_size       125000000;
 	client_body_buffer_size    128k;
 
-	location /$S3_BUCKET {
+	location ~ ^/$S3_BUCKET {
 		internal;
 
 		access_by_lua	 		"sign_aws_request()";

From b9ab699b0191a3619ea08f805d1d32f12c5fcf36 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 16 May 2022 17:21:10 -0400
Subject: [PATCH 30/39] Use versioned ex_aws for B2 compatibility

---
 mix.exs  |  3 ++-
 mix.lock | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/mix.exs b/mix.exs
index aa3ce11e..d1d8ca52 100644
--- a/mix.exs
+++ b/mix.exs
@@ -69,7 +69,8 @@ defmodule Philomena.MixProject do
       {:castore, "~> 0.1"},
       {:mint, "~> 1.2"},
       {:exq, "~> 0.14"},
-      {:ex_aws, "~> 2.0"},
+      {:ex_aws, "~> 2.0",
+       github: "liamwhite/ex_aws", ref: "a340859dd8ac4d63bd7a3948f0994e493e49bda4", override: true},
       {:ex_aws_s3, "~> 2.0"},
       {:sweet_xml, "~> 0.7"},
 
diff --git a/mix.lock b/mix.lock
index 8e0f530e..d9b35788 100644
--- a/mix.lock
+++ b/mix.lock
@@ -7,7 +7,7 @@
   "canada": {:hex, :canada, "1.0.2", "040e4c47609b0a67d5773ac1fbe5e99f840cef173d69b739beda7c98453e0770", [:mix], [], "hexpm", "4269f74153fe89583fe50bd4d5de57bfe01f31258a6b676d296f3681f1483c68"},
   "canary": {:hex, :canary, "1.1.1", "4138d5e05db8497c477e4af73902eb9ae06e49dceaa13c2dd9f0b55525ded48b", [:mix], [{:canada, "~> 1.0.1", [hex: :canada, repo: "hexpm", optional: false]}, {:ecto, ">= 1.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f348d9848693c830a65b707bba9e4dfdd6434e8c356a8d4477e4535afb0d653b"},
   "castore": {:hex, :castore, "0.1.14", "3f6d7c7c1574c402fef29559d3f1a7389ba3524bc6a090a5e9e6abc3af65dcca", [:mix], [], "hexpm", "b34af542eadb727e6c8b37fdf73e18b2e02eb483a4ea0b52fd500bc23f052b7b"},
-  "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
+  "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
@@ -27,18 +27,18 @@
   "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
   "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
-  "ex_aws": {:hex, :ex_aws, "2.2.10", "064139724335b00b6665af7277189afc9ed507791b1ccf2698dadc7c8ad892e8", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "98acb63f74b2f0822be219c5c2f0e8d243c2390f5325ad0557b014d3360da47e"},
+  "ex_aws": {:git, "https://github.com/liamwhite/ex_aws.git", "a340859dd8ac4d63bd7a3948f0994e493e49bda4", [ref: "a340859dd8ac4d63bd7a3948f0994e493e49bda4"]},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.3.2", "92a63b72d763b488510626d528775b26831f5c82b066a63a3128054b7a09de28", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "b235b27131409bcc293c343bf39f1fbdd32892aa237b3f13752e914dc2979960"},
   "exq": {:hex, :exq, "0.16.1", "140d78e95a538d265d23a1e7a090f501c40c799f269b8b503f4cbd962447e708", [:mix], [{:elixir_uuid, ">= 1.2.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0 and < 5.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:redix, ">= 0.9.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "ce70231e2892130b0f80d1bbc8d6ddd22d89d1157b32e783a933eaadb31bc821"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
   "gen_smtp": {:hex, :gen_smtp, "1.1.1", "bf9303c31735100631b1d708d629e4c65944319d1143b5c9952054f4a1311d85", [:rebar3], [{:hut, "1.3.0", [hex: :hut, repo: "hexpm", optional: false]}, {:ranch, ">= 1.7.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "51bc50cc017efd4a4248cbc39ea30fb60efa7d4a49688986fafad84434ff9ab7"},
   "gettext": {:hex, :gettext, "0.19.0", "6909d61b38bb33339558f128f8af5913d5d5fe304a770217bf352b1620fb7ec4", [:mix], [], "hexpm", "3f7a274f52ebda9bb6655dfeda3d6b0dc4537ae51ce41dcccc7f73ca7379ad5e"},
-  "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},
+  "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"},
   "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
   "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"},
-  "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
+  "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
@@ -75,7 +75,7 @@
   "slime": {:hex, :slime, "1.3.0", "153cebb4a837efaf55fb09dff0d79374ad74af835a0288feccbfd9cf606446f9", [:mix], [{:neotoma, "~> 1.7", [hex: :neotoma, repo: "hexpm", optional: false]}], "hexpm", "303b58f05d740a5fe45165bcadfe01da174f1d294069d09ebd7374cd36990a27"},
   "sobelow": {:hex, :sobelow, "0.11.1", "23438964486f8112b41e743bbfd402da3e5b296fdc9eacab29914b79c48916dd", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9897363a7eff96f4809304a90aad819e2ad5e5d24db547af502885146746a53c"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
-  "sweet_xml": {:hex, :sweet_xml, "0.7.2", "4729f997286811fabdd8288f8474e0840a76573051062f066c4b597e76f14f9f", [:mix], [], "hexpm", "6894e68a120f454534d99045ea3325f7740ea71260bc315f82e29731d570a6e8"},
+  "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"},
   "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"},
   "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"},
   "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm", "f1e3dabef71fb510d015fad18c0e05e7c57281001141504c6b69d94e99750a07"},

From 244ea56d0f945eb699154f211bf152b78d15c8d5 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Wed, 18 May 2022 22:45:32 -0400
Subject: [PATCH 31/39] Improve performance of bulk renames

---
 lib/philomena/images/thumbnailer.ex | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex
index b0e6cd1d..6d96256e 100644
--- a/lib/philomena/images/thumbnailer.ex
+++ b/lib/philomena/images/thumbnailer.ex
@@ -134,13 +134,19 @@ defmodule Philomena.Images.Thumbnailer do
   end
 
   defp bulk_rename(file_names, source_prefix, target_prefix) do
-    Enum.map(file_names, fn name ->
-      source = Path.join(source_prefix, name)
-      target = Path.join(target_prefix, name)
+    file_names
+    |> Task.async_stream(
+      fn name ->
+        source = Path.join(source_prefix, name)
+        target = Path.join(target_prefix, name)
+        Objects.copy(source, target)
 
-      Objects.copy(source, target)
-      Objects.delete(source)
-    end)
+        name
+      end,
+      timeout: :infinity
+    )
+    |> Stream.map(fn {:ok, name} -> name end)
+    |> bulk_delete(source_prefix)
   end
 
   defp bulk_delete(file_names, prefix) do

From dd85badae4f21b16530f7dc9be2c2f9ea3b77956 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Thu, 26 May 2022 20:35:45 -0400
Subject: [PATCH 32/39] Faster uploads, fix replacement of existing files

---
 lib/philomena/objects.ex                         | 16 ++++++++++++++++
 lib/philomena/uploader.ex                        |  2 +-
 .../controllers/image/file_controller.ex         |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 2232617f..212bdf92 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -36,6 +36,22 @@ defmodule Philomena.Objects do
     end)
   end
 
+  #
+  # Upload a file using multiple API calls, writing the
+  # contents from the given path to storage.
+  #
+  @spec upload(String.t(), String.t()) :: any()
+  def upload(key, file_path) do
+    {_, mime} = Mime.file(file_path)
+
+    run_all(fn opts ->
+      file_path
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload(opts[:bucket], key, content_type: mime)
+      |> ExAws.request!(opts[:config_overrides])
+    end)
+  end
+
   #
   # Copies a key from the source to the destination,
   # overwriting the destination object if its exists.
diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex
index 76ba4c8d..df982897 100644
--- a/lib/philomena/uploader.ex
+++ b/lib/philomena/uploader.ex
@@ -72,7 +72,7 @@ defmodule Philomena.Uploader do
   content type and permissions.
   """
   def persist_file(path, file) do
-    Objects.put(path, file)
+    Objects.upload(path, file)
   end
 
   @doc """
diff --git a/lib/philomena_web/controllers/image/file_controller.ex b/lib/philomena_web/controllers/image/file_controller.ex
index a8d6175a..8b3f0e3f 100644
--- a/lib/philomena_web/controllers/image/file_controller.ex
+++ b/lib/philomena_web/controllers/image/file_controller.ex
@@ -10,6 +10,8 @@ defmodule PhilomenaWeb.Image.FileController do
   plug PhilomenaWeb.ScraperPlug, params_name: "image", params_key: "image"
 
   def update(conn, %{"image" => image_params}) do
+    Images.remove_hash(conn.assigns.image)
+
     case Images.update_file(conn.assigns.image, image_params) do
       {:ok, image} ->
         conn

From 51b2c7410c01ef6a3eb5adebe63a9f81c46f6861 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Fri, 1 Jul 2022 12:24:51 -0400
Subject: [PATCH 33/39] Spawn off on hides, adjust retries

---
 config/runtime.exs       |  3 +++
 lib/philomena/images.ex  | 10 +++++++---
 lib/philomena/objects.ex |  2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 2e211f30..d5dfc5cb 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -95,6 +95,9 @@ config :ex_aws, :hackney_opts,
   use_default_pool: false,
   pool: false
 
+config :ex_aws, :retries,
+  max_attempts: 20
+
 if config_env() != :test do
   # Database config
   config :philomena, Philomena.Repo,
diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex
index 7ecaf8ff..2ba12f82 100644
--- a/lib/philomena/images.ex
+++ b/lib/philomena/images.ex
@@ -535,14 +535,16 @@ defmodule Philomena.Images do
   defp process_after_hide(result) do
     case result do
       {:ok, %{image: image, tags: tags, reports: {_count, reports}} = result} ->
-        Thumbnailer.hide_thumbnails(image, image.hidden_image_key)
+        spawn(fn ->
+          Thumbnailer.hide_thumbnails(image, image.hidden_image_key)
+          purge_files(image, image.hidden_image_key)
+        end)
 
         Comments.reindex_comments(image)
         Reports.reindex_reports(reports)
         Tags.reindex_tags(tags)
         reindex_image(image)
         reindex_copied_tags(result)
-        purge_files(image, image.hidden_image_key)
 
         {:ok, result}
 
@@ -586,7 +588,9 @@ defmodule Philomena.Images do
     |> Repo.transaction()
     |> case do
       {:ok, %{image: image, tags: tags}} ->
-        Thumbnailer.unhide_thumbnails(image, key)
+        spawn(fn ->
+          Thumbnailer.unhide_thumbnails(image, key)
+        end)
 
         reindex_image(image)
         purge_files(image, image.hidden_image_key)
diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 212bdf92..222c2546 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -47,7 +47,7 @@ defmodule Philomena.Objects do
     run_all(fn opts ->
       file_path
       |> ExAws.S3.Upload.stream_file()
-      |> ExAws.S3.upload(opts[:bucket], key, content_type: mime)
+      |> ExAws.S3.upload(opts[:bucket], key, content_type: mime, max_concurrency: 2)
       |> ExAws.request!(opts[:config_overrides])
     end)
   end

From 6cdcfb2dcd4092f74fbf4905457c567d9f6f7b50 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 18 Jul 2022 10:13:24 -0400
Subject: [PATCH 34/39] Spawn for upload persistence

---
 config/runtime.exs                      |  3 +-
 lib/philomena/images.ex                 | 43 +++++++++++++++++++++++--
 lib/philomena/objects.ex                | 20 ++++++++++--
 lib/philomena_web/plugs/scraper_plug.ex |  2 +-
 4 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index d5dfc5cb..1b46f46d 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -95,8 +95,7 @@ config :ex_aws, :hackney_opts,
   use_default_pool: false,
   pool: false
 
-config :ex_aws, :retries,
-  max_attempts: 20
+config :ex_aws, :retries, max_attempts: 20
 
 if config_env() != :test do
   # Database config
diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex
index 2ba12f82..44403fb8 100644
--- a/lib/philomena/images.ex
+++ b/lib/philomena/images.ex
@@ -4,6 +4,7 @@ defmodule Philomena.Images do
   """
 
   import Ecto.Query, warn: false
+  require Logger
 
   alias Ecto.Multi
   alias Philomena.Repo
@@ -108,9 +109,7 @@ defmodule Philomena.Images do
     |> Repo.transaction()
     |> case do
       {:ok, %{image: image}} = result ->
-        Uploader.persist_upload(image)
-
-        repair_image(image)
+        async_upload(image, attrs["image"])
         reindex_image(image)
         Tags.reindex_tags(image.added_tags)
         maybe_approve_image(image, attribution[:user])
@@ -122,6 +121,44 @@ defmodule Philomena.Images do
     end
   end
 
+  defp async_upload(image, plug_upload) do
+    linked_pid =
+      spawn(fn ->
+        # Make sure task will finish before VM exit
+        Process.flag(:trap_exit, true)
+
+        # Wait to be freed up by the caller
+        receive do
+          :ready -> nil
+        end
+
+        # Start trying to upload
+        try_upload(image, 0)
+      end)
+
+    # Give the upload to the linked process
+    Plug.Upload.give_away(plug_upload, linked_pid, self())
+
+    # Free up the linked process
+    send(linked_pid, :ready)
+  end
+
+  defp try_upload(image, retry_count) when retry_count < 100 do
+    try do
+      Uploader.persist_upload(image)
+      repair_image(image)
+    rescue
+      e ->
+        Logger.error("Upload failed: #{inspect(e)} [try ##{retry_count}]")
+        Process.sleep(5000)
+        try_upload(image, retry_count + 1)
+    end
+  end
+
+  defp try_upload(image, retry_count) do
+    Logger.error("Aborting upload of #{image.id} after #{retry_count} retries")
+  end
+
   defp maybe_create_subscription_on_upload(multi, %User{watch_on_upload: true} = user) do
     multi
     |> Multi.run(:subscribe, fn _repo, %{image: image} ->
diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 222c2546..21f034f6 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -86,10 +86,26 @@ defmodule Philomena.Objects do
     end)
   end
 
-  defp run_all(fun) do
+  defp run_all(wrapped) do
+    fun = fn opts ->
+      try do
+        wrapped.(opts)
+        :ok
+      rescue
+        _ -> :error
+      end
+    end
+
     backends()
     |> Task.async_stream(fun, timeout: :infinity)
-    |> Stream.run()
+    |> Enum.any?(fn {_, v} -> v == :error end)
+    |> case do
+      true ->
+        raise "Failed to operate on all backends"
+
+      _ ->
+        :ok
+    end
   end
 
   defp backends do
diff --git a/lib/philomena_web/plugs/scraper_plug.ex b/lib/philomena_web/plugs/scraper_plug.ex
index 2d47fc56..1fcd0db1 100644
--- a/lib/philomena_web/plugs/scraper_plug.ex
+++ b/lib/philomena_web/plugs/scraper_plug.ex
@@ -34,7 +34,7 @@ defmodule PhilomenaWeb.ScraperPlug do
     params_name = Keyword.get(opts, :params_name, "image")
     params_key = Keyword.get(opts, :params_key, "image")
     name = extract_filename(url, headers)
-    file = Briefly.create!()
+    file = Plug.Upload.random_file!(UUID.uuid1())
 
     File.write!(file, body)
 

From 419ebdfe83c352f1a09bf35f048ef61233108965 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 8 Oct 2022 18:16:01 -0400
Subject: [PATCH 35/39] Remove retries configuration to fix overriding

---
 config/runtime.exs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 1b46f46d..2e211f30 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -95,8 +95,6 @@ config :ex_aws, :hackney_opts,
   use_default_pool: false,
   pool: false
 
-config :ex_aws, :retries, max_attempts: 20
-
 if config_env() != :test do
   # Database config
   config :philomena, Philomena.Repo,

From 464cc26a85b87e968fff15d749e91764d9e90d1b Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 5 Nov 2022 09:23:56 -0400
Subject: [PATCH 36/39] workaround for inconsistent PutObjectCopy on R2

---
 lib/philomena/objects.ex | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 21f034f6..23a55d52 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -3,19 +3,25 @@ defmodule Philomena.Objects do
   Replication wrapper for object storage backends.
   """
   alias Philomena.Mime
+  require Logger
 
   #
-  # Fetch a key from the primary storage backend and
+  # Fetch a key from the storage backend and
   # write it into the destination file.
   #
   # sobelow_skip ["Traversal.FileModule"]
   @spec download_file(String.t(), String.t()) :: any()
   def download_file(key, file_path) do
-    [opts] = primary_opts()
-
     contents =
-      ExAws.S3.get_object(opts[:bucket], key)
-      |> ExAws.request!(opts[:config_overrides])
+      backends()
+      |> Enum.find_value(fn opts ->
+        ExAws.S3.get_object(opts[:bucket], key)
+        |> ExAws.request(opts[:config_overrides])
+        |> case do
+          {:ok, result} -> result
+          _ -> nil
+        end
+      end)
 
     File.write!(file_path, contents.body)
   end
@@ -58,10 +64,16 @@ defmodule Philomena.Objects do
   #
   @spec copy(String.t(), String.t()) :: any()
   def copy(source_key, dest_key) do
-    run_all(fn opts ->
-      ExAws.S3.put_object_copy(opts[:bucket], dest_key, opts[:bucket], source_key)
-      |> ExAws.request!(opts[:config_overrides])
-    end)
+    # Potential workaround for inconsistent PutObjectCopy on R2
+    #
+    # run_all(fn opts->
+    #   ExAws.S3.put_object_copy(opts[:bucket], dest_key, opts[:bucket], source_key)
+    #   |> ExAws.request!(opts[:config_overrides])
+    # end)
+
+    file_path = Briefly.create!()
+    download_file(source_key, file_path)
+    upload(dest_key, file_path)
   end
 
   #
@@ -101,7 +113,7 @@ defmodule Philomena.Objects do
     |> Enum.any?(fn {_, v} -> v == :error end)
     |> case do
       true ->
-        raise "Failed to operate on all backends"
+        Logger.warn("Failed to operate on all backends")
 
       _ ->
         :ok

From a7ffde8f89ee0e0e79b3300fbb3130f35b6b0f88 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 5 Nov 2022 09:44:34 -0400
Subject: [PATCH 37/39] Warn on copy failure

---
 lib/philomena/objects.ex | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index 23a55d52..cfd492d8 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -71,9 +71,13 @@ defmodule Philomena.Objects do
     #   |> ExAws.request!(opts[:config_overrides])
     # end)
 
-    file_path = Briefly.create!()
-    download_file(source_key, file_path)
-    upload(dest_key, file_path)
+    try do
+      file_path = Briefly.create!()
+      download_file(source_key, file_path)
+      upload(dest_key, file_path)
+    rescue
+      _ -> Logger.warn("Failed to copy #{source_key} -> #{dest_key}")
+    end
   end
 
   #

From 1ce934b75ae94b866512cccb92d6795ee9e677d4 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Sat, 5 Nov 2022 11:39:30 -0400
Subject: [PATCH 38/39] Catch more errors, adjust retries

---
 config/runtime.exs       | 5 +++++
 lib/philomena/objects.ex | 8 ++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 2e211f30..883f1345 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -95,6 +95,11 @@ config :ex_aws, :hackney_opts,
   use_default_pool: false,
   pool: false
 
+config :ex_aws, :retries,
+  max_attempts: 20,
+  base_backoff_in_ms: 10,
+  max_backoff_in_ms: 10_000
+
 if config_env() != :test do
   # Database config
   config :philomena, Philomena.Repo,
diff --git a/lib/philomena/objects.ex b/lib/philomena/objects.ex
index cfd492d8..8c18f039 100644
--- a/lib/philomena/objects.ex
+++ b/lib/philomena/objects.ex
@@ -75,8 +75,8 @@ defmodule Philomena.Objects do
       file_path = Briefly.create!()
       download_file(source_key, file_path)
       upload(dest_key, file_path)
-    rescue
-      _ -> Logger.warn("Failed to copy #{source_key} -> #{dest_key}")
+    catch
+      _kind, _value -> Logger.warn("Failed to copy #{source_key} -> #{dest_key}")
     end
   end
 
@@ -107,8 +107,8 @@ defmodule Philomena.Objects do
       try do
         wrapped.(opts)
         :ok
-      rescue
-        _ -> :error
+      catch
+        _kind, _value -> :error
       end
     end
 

From 28523147a438fb40544373bfb156d257ae8adcd7 Mon Sep 17 00:00:00 2001
From: Luna D <luna@nighty.cloud>
Date: Sun, 4 Dec 2022 13:20:18 +0100
Subject: [PATCH 39/39] update s3proxy

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 4e0d6e01..57c429ee 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -77,7 +77,7 @@ services:
       driver: "none"
 
   files:
-    image: andrewgaul/s3proxy:sha-5aec5c1
+    image: andrewgaul/s3proxy:sha-ba0fd6d
     environment:
       - JCLOUDS_FILESYSTEM_BASEDIR=/srv/philomena/priv/s3
     volumes: