Allow processors to indicate the thumbnail types they can generate

config/runtime.exs

@@ -17,7 +17,6 @@ config :philomena,
   elasticsearch_url: System.get_env("ELASTICSEARCH_URL", "http://localhost:9200"),
   advert_file_root: System.fetch_env!("ADVERT_FILE_ROOT"),
   avatar_file_root: System.fetch_env!("AVATAR_FILE_ROOT"),
-  channel_url_root: System.fetch_env!("CHANNEL_URL_ROOT"),
   badge_file_root: System.fetch_env!("BADGE_FILE_ROOT"),
   password_pepper: System.fetch_env!("PASSWORD_PEPPER"),
   avatar_url_root: System.fetch_env!("AVATAR_URL_ROOT"),
@@ -32,6 +31,7 @@ config :philomena,
   tag_url_root: System.fetch_env!("TAG_URL_ROOT"),
   redis_host: System.get_env("REDIS_HOST", "localhost"),
   proxy_host: System.get_env("PROXY_HOST"),
+  s3_bucket: System.fetch_env!("S3_BUCKET")
   camo_host: System.get_env("CAMO_HOST"),
   camo_key: System.get_env("CAMO_KEY"),
   cdn_host: System.fetch_env!("CDN_HOST")

docker-compose.yml

@@ -17,17 +17,16 @@ services:
       - PASSWORD_PEPPER=dn2e0EpZrvBLoxUM3gfQveBhjf0bG/6/bYhrOyq3L3hV9hdo/bimJ+irbDWsuXLP
       - TUMBLR_API_KEY=fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4
       - OTP_SECRET_KEY=Wn7O/8DD+qxL0X4X7bvT90wOkVGcA90bIHww4twR03Ci//zq7PnMw8ypqyyT/b/C
-      - ADVERT_FILE_ROOT=priv/static/system/images/adverts
+      - ADVERT_FILE_ROOT=adverts
-      - AVATAR_FILE_ROOT=priv/static/system/images/avatars
+      - AVATAR_FILE_ROOT=avatars
-      - BADGE_FILE_ROOT=priv/static/system/images
+      - BADGE_FILE_ROOT=badges
-      - IMAGE_FILE_ROOT=priv/static/system/images
+      - IMAGE_FILE_ROOT=images
-      - TAG_FILE_ROOT=priv/static/system/images
+      - TAG_FILE_ROOT=tags
-      - CHANNEL_URL_ROOT=/media
       - AVATAR_URL_ROOT=/avatars
       - ADVERT_URL_ROOT=/spns
       - IMAGE_URL_ROOT=/img
-      - BADGE_URL_ROOT=/media
+      - BADGE_URL_ROOT=/badges
-      - TAG_URL_ROOT=/media
+      - TAG_URL_ROOT=/tags
       - ELASTICSEARCH_URL=http://elasticsearch:9200
       - REDIS_HOST=redis
       - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev

lib/philomena/images/thumbnailer.ex

@@ -10,6 +10,7 @@ defmodule Philomena.Images.Thumbnailer do
   alias Philomena.Analyzers
   alias Philomena.Sha512
   alias Philomena.Repo
+  alias ExAws.S3
 
   @versions [
     thumb_tiny: {50, 50},
@@ -18,27 +19,24 @@ defmodule Philomena.Images.Thumbnailer do
     small: {320, 240},
     medium: {800, 600},
     large: {1280, 1024},
-    tall: {1024, 4096},
+    tall: {1024, 4096}
-    full: nil
   ]
 
   def thumbnail_versions do
-    Enum.filter(@versions, fn {_name, dimensions} ->
+    @versions
-      not is_nil(dimensions)
-    end)
   end
 
-  def thumbnail_urls(image, hidden_key) do
+  # A list of version sizes that should be generated for the image,
-    Path.join([image_thumb_dir(image), "*"])
+  # based on its dimensions. The processor can generate a list of paths.
-    |> Path.wildcard()
+  def generated_sizes(%{image_width: image_width, image_height: image_height}) do
-    |> Enum.map(fn version_name ->
+    Enum.filter(@versions, fn
-      Path.join([image_url_base(image, hidden_key), Path.basename(version_name)])
+      {_name, {width, height}} -> image_width > width or image_height > height
     end)
   end
 
   def generate_thumbnails(image_id) do
     image = Repo.get!(Image, image_id)
-    file = image_file(image)
+    file = download_image_file(image)
     {:ok, analysis} = Analyzers.analyze(file)
 
     apply_edit_script(image, Processors.process(analysis, file, @versions))
@@ -135,6 +133,24 @@ defmodule Philomena.Images.Thumbnailer do
   def image_file(%Image{image: image}),
     do: Path.join(image_file_root(), image)
 
+  defp download_image_file(%Image{image: path} = image) do
+    tempfile = Briefly.create!(extname: "." <> image.image_format)
+    path = Path.join(image_file_root(), path)
+
+    ExAws.request!(S3.download_file(bucket(), path, tempfile))
+
+    tempfile
+  end
+
+  defp upload_image_file(%Image{image: path}, new_file) do
+    path = Path.join(image_file_root(), path)
+
+    new_file
+    |> S3.Upload.stream_file()
+    |> S3.upload(bucket(), path, acl: :public_read)
+    |> ExAws.request!()
+  end
+
   def image_thumb_dir(%Image{
         created_at: created_at,
         id: id,
@@ -163,4 +179,8 @@ defmodule Philomena.Images.Thumbnailer do
 
   defp image_url_root,
     do: Application.get_env(:philomena, :image_url_root)
+
+  defp bucket() do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
 end

lib/philomena/processors.ex

@@ -40,6 +40,15 @@ defmodule Philomena.Processors do
   def processor("video/webm"), do: Webm
   def processor(_content_type), do: nil
 
+  @doc """
+  Takes an analyzer and version list and generates a list of versions to be
+  generated (e.g., ["thumb.png"]). List contents differ based on file type.
+  """
+  @spec versions(map(), keyword) :: [String.t()]
+  def versions(analysis, valid_sizes) do
+    processor(analysis.mime_type).versions(valid_sizes)
+  end
+
   @doc """
   Takes an analyzer, file path, and version list and runs the appropriate
   processor's process/3.

lib/philomena/processors/gif.ex

@@ -1,6 +1,12 @@
 defmodule Philomena.Processors.Gif do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    sizes
+    |> Enum.map(fn {name, _} -> "#{name}.gif" end)
+    |> Kernel.++(["full.webm", "full.mp4", "rendered.png"])
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     duration = analysis.duration

lib/philomena/processors/jpeg.ex

@@ -1,6 +1,10 @@
 defmodule Philomena.Processors.Jpeg do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    Enum.map(sizes, fn {name, _} -> "#{name}.jpg" end)
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     stripped = optimize(strip(file))

lib/philomena/processors/png.ex

@@ -1,6 +1,10 @@
 defmodule Philomena.Processors.Png do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    Enum.map(sizes, fn {name, _} -> "#{name}.png" end)
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     animated? = analysis.animated?

lib/philomena/processors/svg.ex

@@ -1,6 +1,12 @@
 defmodule Philomena.Processors.Svg do
   alias Philomena.Intensities
 
+  def versions(sizes) do
+    sizes
+    |> Enum.map(fn {name, _} -> "#{name}.png" end)
+    |> Kernel.++(["rendered.png", "full.png"])
+  end
+
   def process(analysis, file, versions) do
     preview = preview(file)
 

lib/philomena/processors/webm.ex

@@ -2,6 +2,17 @@ defmodule Philomena.Processors.Webm do
   alias Philomena.Intensities
   import Bitwise
 
+  def versions(sizes) do
+    webm_versions = Enum.map(sizes, fn {name, _} -> "#{name}.webm" end)
+    mp4_versions = Enum.map(sizes, fn {name, _} -> "#{name}.mp4" end)
+    gif_versions =
+      sizes
+      |> Enum.filter(fn {name, _} -> name in [:thumb_tiny, :thumb_small, :thumb] end)
+      |> Enum.map(fn {name, _} -> "#{name}.gif" end)
+
+    webm_versions ++ mp4_versions ++ gif_versions
+  end
+
   def process(analysis, file, versions) do
     dimensions = analysis.dimensions
     duration = analysis.duration

lib/philomena/uploader.ex

@@ -6,6 +6,7 @@ defmodule Philomena.Uploader do
   alias Philomena.Filename
   alias Philomena.Analyzers
   alias Philomena.Sha512
+  alias ExAws.S3
   import Ecto.Changeset
 
   @doc """
@@ -58,18 +59,15 @@ defmodule Philomena.Uploader do
   in the transaction.
   """
   @spec persist_upload(any(), String.t(), String.t()) :: any()
-
-  # sobelow_skip ["Traversal"]
   def persist_upload(model, file_root, field_name) do
     source = Map.get(model, field(upload_key(field_name)))
     dest = Map.get(model, field(field_name))
     target = Path.join(file_root, dest)
-    dir = Path.dirname(target)
 
-    # Create the target directory if it doesn't exist yet,
+    source
-    # then write the file.
+    |> S3.Upload.stream_file()
-    File.mkdir_p!(dir)
+    |> S3.upload(bucket(), target, acl: :public_read)
-    File.cp!(source, target)
+    |> ExAws.request!()
   end
 
   @doc """
@@ -107,8 +105,11 @@ defmodule Philomena.Uploader do
   defp try_remove("", _file_root), do: nil
   defp try_remove(nil, _file_root), do: nil
 
-  # sobelow_skip ["Traversal.FileModule"]
+  defp try_remove(file, file_root) do
-  defp try_remove(file, file_root), do: File.rm(Path.join(file_root, file))
+    path = Path.join(file_root, file)
+
+    ExAws.request!(S3.delete_object(bucket(), path))
+  end
 
   defp prefix_attributes(map, prefix),
     do: Map.new(map, fn {key, value} -> {"#{prefix}_#{key}", value} end)
@@ -118,4 +119,8 @@ defmodule Philomena.Uploader do
   defp remove_key(field_name), do: "removed_#{field_name}"
 
   defp field(field_name), do: String.to_existing_atom(field_name)
+
+  defp bucket do
+    Application.fetch_env!(:philomena, :s3_bucket)
+  end
 end