From 6e4771a57aefc0106cee0e4975230d4135586e8f Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 7 Feb 2022 22:12:40 -0500 Subject: [PATCH] Allow processors to indicate the thumbnail types they can generate --- config/runtime.exs | 2 +- docker-compose.yml | 15 +++++------ lib/philomena/images/thumbnailer.ex | 42 +++++++++++++++++++++-------- lib/philomena/processors.ex | 9 +++++++ lib/philomena/processors/gif.ex | 6 +++++ lib/philomena/processors/jpeg.ex | 4 +++ lib/philomena/processors/png.ex | 4 +++ lib/philomena/processors/svg.ex | 6 +++++ lib/philomena/processors/webm.ex | 11 ++++++++ lib/philomena/uploader.ex | 23 +++++++++------- 10 files changed, 93 insertions(+), 29 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index bece7870..9605d362 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -17,7 +17,6 @@ config :philomena, elasticsearch_url: System.get_env("ELASTICSEARCH_URL", "http://localhost:9200"), advert_file_root: System.fetch_env!("ADVERT_FILE_ROOT"), avatar_file_root: System.fetch_env!("AVATAR_FILE_ROOT"), - channel_url_root: System.fetch_env!("CHANNEL_URL_ROOT"), badge_file_root: System.fetch_env!("BADGE_FILE_ROOT"), password_pepper: System.fetch_env!("PASSWORD_PEPPER"), avatar_url_root: System.fetch_env!("AVATAR_URL_ROOT"), @@ -32,6 +31,7 @@ config :philomena, tag_url_root: System.fetch_env!("TAG_URL_ROOT"), redis_host: System.get_env("REDIS_HOST", "localhost"), proxy_host: System.get_env("PROXY_HOST"), + s3_bucket: System.fetch_env!("S3_BUCKET") camo_host: System.get_env("CAMO_HOST"), camo_key: System.get_env("CAMO_KEY"), cdn_host: System.fetch_env!("CDN_HOST") diff --git a/docker-compose.yml b/docker-compose.yml index 84c6d7c9..9a27584b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,17 +17,16 @@ services: - PASSWORD_PEPPER=dn2e0EpZrvBLoxUM3gfQveBhjf0bG/6/bYhrOyq3L3hV9hdo/bimJ+irbDWsuXLP - TUMBLR_API_KEY=fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4 - OTP_SECRET_KEY=Wn7O/8DD+qxL0X4X7bvT90wOkVGcA90bIHww4twR03Ci//zq7PnMw8ypqyyT/b/C - - ADVERT_FILE_ROOT=priv/static/system/images/adverts - - AVATAR_FILE_ROOT=priv/static/system/images/avatars - - BADGE_FILE_ROOT=priv/static/system/images - - IMAGE_FILE_ROOT=priv/static/system/images - - TAG_FILE_ROOT=priv/static/system/images - - CHANNEL_URL_ROOT=/media + - ADVERT_FILE_ROOT=adverts + - AVATAR_FILE_ROOT=avatars + - BADGE_FILE_ROOT=badges + - IMAGE_FILE_ROOT=images + - TAG_FILE_ROOT=tags - AVATAR_URL_ROOT=/avatars - ADVERT_URL_ROOT=/spns - IMAGE_URL_ROOT=/img - - BADGE_URL_ROOT=/media - - TAG_URL_ROOT=/media + - BADGE_URL_ROOT=/badges + - TAG_URL_ROOT=/tags - ELASTICSEARCH_URL=http://elasticsearch:9200 - REDIS_HOST=redis - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex index 97da4083..c1c8cdeb 100644 --- a/lib/philomena/images/thumbnailer.ex +++ b/lib/philomena/images/thumbnailer.ex @@ -10,6 +10,7 @@ defmodule Philomena.Images.Thumbnailer do alias Philomena.Analyzers alias Philomena.Sha512 alias Philomena.Repo + alias ExAws.S3 @versions [ thumb_tiny: {50, 50}, @@ -18,27 +19,24 @@ defmodule Philomena.Images.Thumbnailer do small: {320, 240}, medium: {800, 600}, large: {1280, 1024}, - tall: {1024, 4096}, - full: nil + tall: {1024, 4096} ] def thumbnail_versions do - Enum.filter(@versions, fn {_name, dimensions} -> - not is_nil(dimensions) - end) + @versions end - def thumbnail_urls(image, hidden_key) do - Path.join([image_thumb_dir(image), "*"]) - |> Path.wildcard() - |> Enum.map(fn version_name -> - Path.join([image_url_base(image, hidden_key), Path.basename(version_name)]) + # A list of version sizes that should be generated for the image, + # based on its dimensions. The processor can generate a list of paths. + def generated_sizes(%{image_width: image_width, image_height: image_height}) do + Enum.filter(@versions, fn + {_name, {width, height}} -> image_width > width or image_height > height end) end def generate_thumbnails(image_id) do image = Repo.get!(Image, image_id) - file = image_file(image) + file = download_image_file(image) {:ok, analysis} = Analyzers.analyze(file) apply_edit_script(image, Processors.process(analysis, file, @versions)) @@ -135,6 +133,24 @@ defmodule Philomena.Images.Thumbnailer do def image_file(%Image{image: image}), do: Path.join(image_file_root(), image) + defp download_image_file(%Image{image: path} = image) do + tempfile = Briefly.create!(extname: "." <> image.image_format) + path = Path.join(image_file_root(), path) + + ExAws.request!(S3.download_file(bucket(), path, tempfile)) + + tempfile + end + + defp upload_image_file(%Image{image: path}, new_file) do + path = Path.join(image_file_root(), path) + + new_file + |> S3.Upload.stream_file() + |> S3.upload(bucket(), path, acl: :public_read) + |> ExAws.request!() + end + def image_thumb_dir(%Image{ created_at: created_at, id: id, @@ -163,4 +179,8 @@ defmodule Philomena.Images.Thumbnailer do defp image_url_root, do: Application.get_env(:philomena, :image_url_root) + + defp bucket() do + Application.fetch_env!(:philomena, :s3_bucket) + end end diff --git a/lib/philomena/processors.ex b/lib/philomena/processors.ex index 0a6b4afb..531d0533 100644 --- a/lib/philomena/processors.ex +++ b/lib/philomena/processors.ex @@ -40,6 +40,15 @@ defmodule Philomena.Processors do def processor("video/webm"), do: Webm def processor(_content_type), do: nil + @doc """ + Takes an analyzer and version list and generates a list of versions to be + generated (e.g., ["thumb.png"]). List contents differ based on file type. + """ + @spec versions(map(), keyword) :: [String.t()] + def versions(analysis, valid_sizes) do + processor(analysis.mime_type).versions(valid_sizes) + end + @doc """ Takes an analyzer, file path, and version list and runs the appropriate processor's process/3. diff --git a/lib/philomena/processors/gif.ex b/lib/philomena/processors/gif.ex index 7e1f1b36..757d6968 100644 --- a/lib/philomena/processors/gif.ex +++ b/lib/philomena/processors/gif.ex @@ -1,6 +1,12 @@ defmodule Philomena.Processors.Gif do alias Philomena.Intensities + def versions(sizes) do + sizes + |> Enum.map(fn {name, _} -> "#{name}.gif" end) + |> Kernel.++(["full.webm", "full.mp4", "rendered.png"]) + end + def process(analysis, file, versions) do dimensions = analysis.dimensions duration = analysis.duration diff --git a/lib/philomena/processors/jpeg.ex b/lib/philomena/processors/jpeg.ex index e4dde2ff..18257d9f 100644 --- a/lib/philomena/processors/jpeg.ex +++ b/lib/philomena/processors/jpeg.ex @@ -1,6 +1,10 @@ defmodule Philomena.Processors.Jpeg do alias Philomena.Intensities + def versions(sizes) do + Enum.map(sizes, fn {name, _} -> "#{name}.jpg" end) + end + def process(analysis, file, versions) do dimensions = analysis.dimensions stripped = optimize(strip(file)) diff --git a/lib/philomena/processors/png.ex b/lib/philomena/processors/png.ex index 90e739ae..91d2c87f 100644 --- a/lib/philomena/processors/png.ex +++ b/lib/philomena/processors/png.ex @@ -1,6 +1,10 @@ defmodule Philomena.Processors.Png do alias Philomena.Intensities + def versions(sizes) do + Enum.map(sizes, fn {name, _} -> "#{name}.png" end) + end + def process(analysis, file, versions) do dimensions = analysis.dimensions animated? = analysis.animated? diff --git a/lib/philomena/processors/svg.ex b/lib/philomena/processors/svg.ex index 0a16d795..8a9c332b 100644 --- a/lib/philomena/processors/svg.ex +++ b/lib/philomena/processors/svg.ex @@ -1,6 +1,12 @@ defmodule Philomena.Processors.Svg do alias Philomena.Intensities + def versions(sizes) do + sizes + |> Enum.map(fn {name, _} -> "#{name}.png" end) + |> Kernel.++(["rendered.png", "full.png"]) + end + def process(analysis, file, versions) do preview = preview(file) diff --git a/lib/philomena/processors/webm.ex b/lib/philomena/processors/webm.ex index 342135ce..b9da1277 100644 --- a/lib/philomena/processors/webm.ex +++ b/lib/philomena/processors/webm.ex @@ -2,6 +2,17 @@ defmodule Philomena.Processors.Webm do alias Philomena.Intensities import Bitwise + def versions(sizes) do + webm_versions = Enum.map(sizes, fn {name, _} -> "#{name}.webm" end) + mp4_versions = Enum.map(sizes, fn {name, _} -> "#{name}.mp4" end) + gif_versions = + sizes + |> Enum.filter(fn {name, _} -> name in [:thumb_tiny, :thumb_small, :thumb] end) + |> Enum.map(fn {name, _} -> "#{name}.gif" end) + + webm_versions ++ mp4_versions ++ gif_versions + end + def process(analysis, file, versions) do dimensions = analysis.dimensions duration = analysis.duration diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex index 4dc765ac..effb31a3 100644 --- a/lib/philomena/uploader.ex +++ b/lib/philomena/uploader.ex @@ -6,6 +6,7 @@ defmodule Philomena.Uploader do alias Philomena.Filename alias Philomena.Analyzers alias Philomena.Sha512 + alias ExAws.S3 import Ecto.Changeset @doc """ @@ -58,18 +59,15 @@ defmodule Philomena.Uploader do in the transaction. """ @spec persist_upload(any(), String.t(), String.t()) :: any() - - # sobelow_skip ["Traversal"] def persist_upload(model, file_root, field_name) do source = Map.get(model, field(upload_key(field_name))) dest = Map.get(model, field(field_name)) target = Path.join(file_root, dest) - dir = Path.dirname(target) - # Create the target directory if it doesn't exist yet, - # then write the file. - File.mkdir_p!(dir) - File.cp!(source, target) + source + |> S3.Upload.stream_file() + |> S3.upload(bucket(), target, acl: :public_read) + |> ExAws.request!() end @doc """ @@ -107,8 +105,11 @@ defmodule Philomena.Uploader do defp try_remove("", _file_root), do: nil defp try_remove(nil, _file_root), do: nil - # sobelow_skip ["Traversal.FileModule"] - defp try_remove(file, file_root), do: File.rm(Path.join(file_root, file)) + defp try_remove(file, file_root) do + path = Path.join(file_root, file) + + ExAws.request!(S3.delete_object(bucket(), path)) + end defp prefix_attributes(map, prefix), do: Map.new(map, fn {key, value} -> {"#{prefix}_#{key}", value} end) @@ -118,4 +119,8 @@ defmodule Philomena.Uploader do defp remove_key(field_name), do: "removed_#{field_name}" defp field(field_name), do: String.to_existing_atom(field_name) + + defp bucket do + Application.fetch_env!(:philomena, :s3_bucket) + end end