mirror of
https://github.com/philomena-dev/philomena.git
synced 2024-11-30 14:57:59 +01:00
refactor processor architecture
This commit is contained in:
parent
e00b16ab74
commit
8a8281eaba
12 changed files with 424 additions and 242 deletions
|
@ -23,11 +23,15 @@ services:
|
||||||
image: postgres:12.1
|
image: postgres:12.1
|
||||||
volumes:
|
volumes:
|
||||||
- postgres_data:/var/lib/postgresql/data
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
logging:
|
||||||
|
driver: "none"
|
||||||
|
|
||||||
elasticsearch:
|
elasticsearch:
|
||||||
image: elasticsearch:6.8.5
|
image: elasticsearch:6.8.5
|
||||||
volumes:
|
volumes:
|
||||||
- elastic_data:/var/lib/elasticsearch
|
- elastic_data:/var/lib/elasticsearch
|
||||||
|
logging:
|
||||||
|
driver: "none"
|
||||||
ulimits:
|
ulimits:
|
||||||
nofile:
|
nofile:
|
||||||
soft: 65536
|
soft: 65536
|
||||||
|
@ -35,6 +39,8 @@ services:
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:5.0.7
|
image: redis:5.0.7
|
||||||
|
logging:
|
||||||
|
driver: "none"
|
||||||
|
|
||||||
web:
|
web:
|
||||||
build:
|
build:
|
||||||
|
@ -42,6 +48,8 @@ services:
|
||||||
dockerfile: ./docker/web/Dockerfile
|
dockerfile: ./docker/web/Dockerfile
|
||||||
volumes:
|
volumes:
|
||||||
- .:/srv/philomena
|
- .:/srv/philomena
|
||||||
|
logging:
|
||||||
|
driver: "none"
|
||||||
depends_on:
|
depends_on:
|
||||||
- app
|
- app
|
||||||
ports:
|
ports:
|
||||||
|
|
55
lib/philomena/analyzers.ex
Normal file
55
lib/philomena/analyzers.ex
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
defmodule Philomena.Analyzers do
|
||||||
|
@moduledoc """
|
||||||
|
Utilities for analyzing the format and various attributes of uploaded files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias Philomena.Mime
|
||||||
|
|
||||||
|
alias Philomena.Analyzers.Gif
|
||||||
|
alias Philomena.Analyzers.Jpeg
|
||||||
|
alias Philomena.Analyzers.Png
|
||||||
|
alias Philomena.Analyzers.Svg
|
||||||
|
alias Philomena.Analyzers.Webm
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Returns an {:ok, analyzer} tuple, with the analyzer being a module capable
|
||||||
|
of analyzing this content type, or :error.
|
||||||
|
|
||||||
|
To use an analyzer, call the analyze/1 method on it with the path to the
|
||||||
|
file. It will return a map such as the following:
|
||||||
|
|
||||||
|
%{
|
||||||
|
animated?: false,
|
||||||
|
dimensions: {800, 600},
|
||||||
|
duration: 0.0,
|
||||||
|
extension: "png",
|
||||||
|
mime_type: "image/png"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
@spec analyzer(binary()) :: {:ok, module()} | :error
|
||||||
|
def analyzer(content_type)
|
||||||
|
|
||||||
|
def analyzer("image/gif"), do: {:ok, Gif}
|
||||||
|
def analyzer("image/jpeg"), do: {:ok, Jpeg}
|
||||||
|
def analyzer("image/png"), do: {:ok, Png}
|
||||||
|
def analyzer("image/svg+xml"), do: {:ok, Svg}
|
||||||
|
def analyzer("video/webm"), do: {:ok, Webm}
|
||||||
|
def analyzer(_content_type), do: :error
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Attempts a mime check and analysis on the given pathname or Plug.Upload.
|
||||||
|
"""
|
||||||
|
@spec analyze(Plug.Upload.t() | String.t()) :: {:ok, map()} | :error
|
||||||
|
def analyze(%Plug.Upload{path: path}), do: analyze(path)
|
||||||
|
def analyze(path) when is_binary(path) do
|
||||||
|
with {:ok, mime} <- Mime.file(path),
|
||||||
|
{:ok, analyzer} <- analyzer(mime)
|
||||||
|
do
|
||||||
|
{:ok, analyzer.analyze(path)}
|
||||||
|
else
|
||||||
|
error ->
|
||||||
|
error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
def analyze(_path), do: :error
|
||||||
|
end
|
35
lib/philomena/filename.ex
Normal file
35
lib/philomena/filename.ex
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
defmodule Philomena.Filename do
|
||||||
|
@moduledoc """
|
||||||
|
Utilities for building arbitrary filenames for uploaded files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@spec build(String.t()) :: String.t()
|
||||||
|
def build(extension) do
|
||||||
|
[
|
||||||
|
time_identifier(DateTime.utc_now()),
|
||||||
|
"/",
|
||||||
|
usec_identifier(),
|
||||||
|
pid_identifier(),
|
||||||
|
".",
|
||||||
|
extension
|
||||||
|
]
|
||||||
|
|> Enum.join()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp time_identifier(time) do
|
||||||
|
Enum.join([time.year, time.month, time.day], "/")
|
||||||
|
end
|
||||||
|
|
||||||
|
defp usec_identifier do
|
||||||
|
DateTime.utc_now()
|
||||||
|
|> DateTime.to_unix(:microsecond)
|
||||||
|
|> to_string()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp pid_identifier do
|
||||||
|
self()
|
||||||
|
|> :erlang.pid_to_list()
|
||||||
|
|> to_string()
|
||||||
|
|> String.replace(~r/[^0-9]/, "")
|
||||||
|
end
|
||||||
|
end
|
|
@ -9,11 +9,11 @@ defmodule Philomena.Images do
|
||||||
alias Philomena.Repo
|
alias Philomena.Repo
|
||||||
|
|
||||||
alias Philomena.Images.Image
|
alias Philomena.Images.Image
|
||||||
|
alias Philomena.Images.Uploader
|
||||||
alias Philomena.SourceChanges.SourceChange
|
alias Philomena.SourceChanges.SourceChange
|
||||||
alias Philomena.TagChanges.TagChange
|
alias Philomena.TagChanges.TagChange
|
||||||
alias Philomena.Tags
|
alias Philomena.Tags
|
||||||
alias Philomena.Tags.Tag
|
alias Philomena.Tags.Tag
|
||||||
alias Philomena.Processors
|
|
||||||
alias Philomena.Notifications
|
alias Philomena.Notifications
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
|
@ -53,7 +53,7 @@ defmodule Philomena.Images do
|
||||||
%Image{}
|
%Image{}
|
||||||
|> Image.creation_changeset(attrs, attribution)
|
|> Image.creation_changeset(attrs, attribution)
|
||||||
|> Image.tag_changeset(attrs, [], tags)
|
|> Image.tag_changeset(attrs, [], tags)
|
||||||
|> Processors.after_upload(attrs)
|
|> Uploader.analyze_upload(attrs)
|
||||||
|
|
||||||
Multi.new
|
Multi.new
|
||||||
|> Multi.insert(:image, image)
|
|> Multi.insert(:image, image)
|
||||||
|
@ -74,7 +74,7 @@ defmodule Philomena.Images do
|
||||||
create_subscription(image, attribution[:user])
|
create_subscription(image, attribution[:user])
|
||||||
end)
|
end)
|
||||||
|> Multi.run(:after, fn _repo, %{image: image} ->
|
|> Multi.run(:after, fn _repo, %{image: image} ->
|
||||||
Processors.after_insert(image)
|
Uploader.persist_upload(image)
|
||||||
|
|
||||||
{:ok, nil}
|
{:ok, nil}
|
||||||
end)
|
end)
|
||||||
|
|
|
@ -53,6 +53,7 @@ defmodule Philomena.Images.Image do
|
||||||
field :image_format, :string
|
field :image_format, :string
|
||||||
field :image_mime_type, :string
|
field :image_mime_type, :string
|
||||||
field :image_aspect_ratio, :float
|
field :image_aspect_ratio, :float
|
||||||
|
field :image_is_animated, :boolean, source: :is_animated
|
||||||
field :ip, EctoNetwork.INET
|
field :ip, EctoNetwork.INET
|
||||||
field :fingerprint, :string
|
field :fingerprint, :string
|
||||||
field :user_agent, :string, default: ""
|
field :user_agent, :string, default: ""
|
||||||
|
@ -77,7 +78,6 @@ defmodule Philomena.Images.Image do
|
||||||
field :tag_editing_allowed, :boolean, default: true
|
field :tag_editing_allowed, :boolean, default: true
|
||||||
field :description_editing_allowed, :boolean, default: true
|
field :description_editing_allowed, :boolean, default: true
|
||||||
field :commenting_allowed, :boolean, default: true
|
field :commenting_allowed, :boolean, default: true
|
||||||
field :is_animated, :boolean
|
|
||||||
field :first_seen_at, :naive_datetime
|
field :first_seen_at, :naive_datetime
|
||||||
field :destroyed_content, :boolean
|
field :destroyed_content, :boolean
|
||||||
field :hidden_image_key, :string
|
field :hidden_image_key, :string
|
||||||
|
@ -129,13 +129,13 @@ defmodule Philomena.Images.Image do
|
||||||
:image, :image_name, :image_width, :image_height, :image_size,
|
:image, :image_name, :image_width, :image_height, :image_size,
|
||||||
:image_format, :image_mime_type, :image_aspect_ratio,
|
:image_format, :image_mime_type, :image_aspect_ratio,
|
||||||
:image_orig_sha512_hash, :image_sha512_hash, :uploaded_image,
|
:image_orig_sha512_hash, :image_sha512_hash, :uploaded_image,
|
||||||
:is_animated
|
:image_is_animated
|
||||||
])
|
])
|
||||||
|> validate_required([
|
|> validate_required([
|
||||||
:image, :image_width, :image_height, :image_size,
|
:image, :image_width, :image_height, :image_size,
|
||||||
:image_format, :image_mime_type, :image_aspect_ratio,
|
:image_format, :image_mime_type, :image_aspect_ratio,
|
||||||
:image_orig_sha512_hash, :image_sha512_hash, :uploaded_image,
|
:image_orig_sha512_hash, :image_sha512_hash, :uploaded_image,
|
||||||
:is_animated
|
:image_is_animated
|
||||||
])
|
])
|
||||||
|> validate_number(:image_size, greater_than: 0, less_than_or_equal_to: 26214400)
|
|> validate_number(:image_size, greater_than: 0, less_than_or_equal_to: 26214400)
|
||||||
|> validate_number(:image_width, greater_than: 0, less_than_or_equal_to: 32767)
|
|> validate_number(:image_width, greater_than: 0, less_than_or_equal_to: 32767)
|
||||||
|
|
122
lib/philomena/images/thumbnailer.ex
Normal file
122
lib/philomena/images/thumbnailer.ex
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
defmodule Philomena.Images.Thumbnailer do
|
||||||
|
@moduledoc """
|
||||||
|
Prevewing and thumbnailing logic for Images.
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias Philomena.DuplicateReports
|
||||||
|
alias Philomena.ImageIntensities
|
||||||
|
alias Philomena.Images.Image
|
||||||
|
alias Philomena.Processors
|
||||||
|
alias Philomena.Analyzers
|
||||||
|
alias Philomena.Sha512
|
||||||
|
alias Philomena.Repo
|
||||||
|
|
||||||
|
@versions [
|
||||||
|
thumb_tiny: {50, 50},
|
||||||
|
thumb_small: {150, 150},
|
||||||
|
thumb: {250, 250},
|
||||||
|
small: {320, 240},
|
||||||
|
medium: {800, 600},
|
||||||
|
large: {1280, 1024},
|
||||||
|
tall: {1024, 4096},
|
||||||
|
full: nil
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_thumbnails(image_id) do
|
||||||
|
image = Repo.get!(Image, image_id)
|
||||||
|
file = image_file(image)
|
||||||
|
{:ok, analysis} = Analyzers.analyze(file)
|
||||||
|
|
||||||
|
apply_edit_script(image, Processors.process(analysis, file, @versions))
|
||||||
|
recompute_sha512(image, file, &Image.thumbnail_changeset/2)
|
||||||
|
generate_dupe_reports(image)
|
||||||
|
|
||||||
|
apply_edit_script(image, Processors.post_process(analysis, file))
|
||||||
|
recompute_sha512(image, file, &Image.process_changeset/2)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
defp apply_edit_script(image, changes),
|
||||||
|
do: Enum.map(changes, &apply_change(image, &1))
|
||||||
|
|
||||||
|
|
||||||
|
defp apply_change(image, {:intensities, intensities}),
|
||||||
|
do: ImageIntensities.create_image_intensity(image, intensities)
|
||||||
|
|
||||||
|
defp apply_change(image, {:replace_original, new_file}),
|
||||||
|
do: copy(new_file, image_file(image))
|
||||||
|
|
||||||
|
defp apply_change(image, {:thumbnails, thumbnails}),
|
||||||
|
do: Enum.map(thumbnails, &apply_thumbnail(image, image_thumb_dir(image), &1))
|
||||||
|
|
||||||
|
|
||||||
|
defp apply_thumbnail(_image, thumb_dir, {:copy, new_file, destination}),
|
||||||
|
do: copy(new_file, Path.join(thumb_dir, destination))
|
||||||
|
|
||||||
|
defp apply_thumbnail(image, thumb_dir, {:symlink_original, destination}),
|
||||||
|
do: symlink(image_file(image), Path.join(thumb_dir, destination))
|
||||||
|
|
||||||
|
|
||||||
|
defp recompute_sha512(image, file, changeset_fn),
|
||||||
|
do: Repo.update!(changeset_fn.(image, %{"image_sha512_hash" => Sha512.file(file)}))
|
||||||
|
|
||||||
|
defp generate_dupe_reports(image),
|
||||||
|
do: DuplicateReports.generate_reports(image)
|
||||||
|
|
||||||
|
|
||||||
|
# Copy from source to destination, creating parent directories along
|
||||||
|
# the way and setting the appropriate permission bits when necessary.
|
||||||
|
defp copy(source, destination) do
|
||||||
|
prepare_dir(destination)
|
||||||
|
|
||||||
|
File.rm(destination)
|
||||||
|
File.cp!(source, destination)
|
||||||
|
|
||||||
|
set_perms(destination)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Try to handle filesystems that don't support symlinks
|
||||||
|
# by falling back to a copy.
|
||||||
|
defp symlink(source, destination) do
|
||||||
|
source = Path.absname(source)
|
||||||
|
|
||||||
|
prepare_dir(destination)
|
||||||
|
|
||||||
|
case File.ln_s(source, destination) do
|
||||||
|
:ok ->
|
||||||
|
set_perms(destination)
|
||||||
|
|
||||||
|
_err ->
|
||||||
|
copy(source, destination)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# 0o644 = (S_IRUSR | S_IWUSR) | S_IRGRP | S_IROTH
|
||||||
|
defp set_perms(destination),
|
||||||
|
do: File.chmod(destination, 0o644)
|
||||||
|
|
||||||
|
# Prepare the directory by creating it if it does not yet exist.
|
||||||
|
defp prepare_dir(destination) do
|
||||||
|
destination
|
||||||
|
|> Path.dirname()
|
||||||
|
|> File.mkdir_p!()
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
defp image_file(%Image{image: image}),
|
||||||
|
do: Path.join(image_file_root(), image)
|
||||||
|
|
||||||
|
defp image_thumb_dir(%Image{created_at: created_at, id: id}),
|
||||||
|
do: Path.join([image_thumbnail_root(), time_identifier(created_at), to_string(id)])
|
||||||
|
|
||||||
|
defp time_identifier(time),
|
||||||
|
do: Enum.join([time.year, time.month, time.day], "/")
|
||||||
|
|
||||||
|
|
||||||
|
defp image_file_root,
|
||||||
|
do: Application.get_env(:philomena, :image_file_root)
|
||||||
|
|
||||||
|
defp image_thumbnail_root,
|
||||||
|
do: Application.get_env(:philomena, :image_file_root) <> "/thumbs"
|
||||||
|
end
|
20
lib/philomena/images/uploader.ex
Normal file
20
lib/philomena/images/uploader.ex
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
defmodule Philomena.Images.Uploader do
|
||||||
|
@moduledoc """
|
||||||
|
Upload and processing callback logic for Images.
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias Philomena.Images.Image
|
||||||
|
alias Philomena.Uploader
|
||||||
|
|
||||||
|
def analyze_upload(image, params) do
|
||||||
|
Uploader.analyze_upload(image, "image", params["image"], &Image.image_changeset/2)
|
||||||
|
end
|
||||||
|
|
||||||
|
def persist_upload(image) do
|
||||||
|
Uploader.persist_upload(image, image_file_root(), "image")
|
||||||
|
end
|
||||||
|
|
||||||
|
defp image_file_root do
|
||||||
|
Application.get_env(:philomena, :image_file_root)
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,16 +1,37 @@
|
||||||
defmodule Philomena.Mime do
|
defmodule Philomena.Mime do
|
||||||
|
@type mime :: String.t()
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
Gets the MIME type of the given pathname.
|
Gets the mime type of the given pathname.
|
||||||
"""
|
"""
|
||||||
@spec file(String.t()) :: {:ok, binary()} | :error
|
@spec file(String.t()) :: {:ok, mime()} | :error
|
||||||
def file(path) do
|
def file(path) do
|
||||||
System.cmd("file", ["-b", "--mime-type", path])
|
System.cmd("file", ["-b", "--mime-type", path])
|
||||||
|> case do
|
|> case do
|
||||||
{output, 0} ->
|
{output, 0} ->
|
||||||
{:ok, String.trim(output)}
|
true_mime(String.trim(output))
|
||||||
|
|
||||||
_error ->
|
_error ->
|
||||||
:error
|
:error
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Provides the "true" content type of this file.
|
||||||
|
|
||||||
|
Some files are identified incorrectly as a mime type they should not be.
|
||||||
|
These incorrect mime types (and their "corrected") versions are:
|
||||||
|
|
||||||
|
- image/svg -> image/svg+xml
|
||||||
|
- audio/webm -> video/webm
|
||||||
|
"""
|
||||||
|
@spec true_mime(String.t()) :: {:ok, mime()}
|
||||||
|
def true_mime("image/svg"), do: {:ok, "image/svg+xml"}
|
||||||
|
def true_mime("audio/webm"), do: {:ok, "video/webm"}
|
||||||
|
|
||||||
|
def true_mime(mime)
|
||||||
|
when mime in ~W(image/gif image/jpeg image/png image/svg+xml video/webm),
|
||||||
|
do: {:ok, mime}
|
||||||
|
|
||||||
|
def true_mime(_mime), do: :error
|
||||||
end
|
end
|
|
@ -1,225 +1,69 @@
|
||||||
defmodule Philomena.Processors do
|
defmodule Philomena.Processors do
|
||||||
alias Philomena.Images.Image
|
@moduledoc """
|
||||||
alias Philomena.DuplicateReports
|
Utilities for processing uploads.
|
||||||
alias Philomena.ImageIntensities
|
|
||||||
alias Philomena.Repo
|
|
||||||
alias Philomena.Mime
|
|
||||||
alias Philomena.Sha512
|
|
||||||
|
|
||||||
def mimes(type) do
|
Processors have 3 methods available:
|
||||||
%{
|
|
||||||
"image/gif" => "image/gif",
|
- process/3:
|
||||||
"image/jpeg" => "image/jpeg",
|
Takes an analysis, file path, and version list and generates an
|
||||||
"image/png" => "image/png",
|
"edit script" that represents how to store this file according to the
|
||||||
"image/svg+xml" => "image/svg+xml",
|
given version list. See Philomena.Images.Thumbnailer for more
|
||||||
"video/webm" => "video/webm",
|
information on how this works.
|
||||||
"image/svg" => "image/svg+xml",
|
|
||||||
"audio/webm" => "video/webm"
|
- post_process/2:
|
||||||
}
|
Takes an analysis and file path and performs optimizations on the
|
||||||
|> Map.get(type)
|
upload. See Philomena.Images.Thumbnailer for more information on how this
|
||||||
|
works.
|
||||||
|
|
||||||
|
- intensities/2:
|
||||||
|
Takes an analysis and file path and generates an intensities map
|
||||||
|
appropriate for use by Philomena.DuplicateReports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias Philomena.Processors.Gif
|
||||||
|
alias Philomena.Processors.Jpeg
|
||||||
|
alias Philomena.Processors.Png
|
||||||
|
alias Philomena.Processors.Svg
|
||||||
|
alias Philomena.Processors.Webm
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Returns a processor, with the processor being a module capable
|
||||||
|
of processing this content type, or nil.
|
||||||
|
"""
|
||||||
|
@spec processor(String.t()) :: module() | nil
|
||||||
|
def processor(content_type)
|
||||||
|
|
||||||
|
def processor("image/gif"), do: Gif
|
||||||
|
def processor("image/jpeg"), do: Jpeg
|
||||||
|
def processor("image/png"), do: Png
|
||||||
|
def processor("image/svg+xml"), do: Svg
|
||||||
|
def processor("video/webm"), do: Webm
|
||||||
|
def processor(_content_type), do: nil
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Takes an analyzer, file path, and version list and runs the appropriate
|
||||||
|
processor's process/3.
|
||||||
|
"""
|
||||||
|
@spec process(map(), String.t(), keyword) :: map()
|
||||||
|
def process(analysis, file, versions) do
|
||||||
|
processor(analysis.mime_type).process(analysis, file, versions)
|
||||||
end
|
end
|
||||||
|
|
||||||
def analyzers(type) do
|
@doc """
|
||||||
%{
|
Takes an analyzer and file path and runs the appropriate processor's
|
||||||
"image/gif" => Philomena.Analyzers.Gif,
|
post_process/2.
|
||||||
"image/jpeg" => Philomena.Analyzers.Jpeg,
|
"""
|
||||||
"image/png" => Philomena.Analyzers.Png,
|
@spec post_process(map(), String.t()) :: map()
|
||||||
"image/svg+xml" => Philomena.Analyzers.Svg,
|
def post_process(analysis, file) do
|
||||||
"video/webm" => Philomena.Analyzers.Webm
|
processor(analysis.mime_type).post_process(analysis, file)
|
||||||
}
|
|
||||||
|> Map.get(type)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def processors(type) do
|
@doc """
|
||||||
%{
|
Takes an analyzer and file path and runs the appropriate processor's
|
||||||
"image/gif" => Philomena.Processors.Gif,
|
intensities/2.
|
||||||
"image/jpeg" => Philomena.Processors.Jpeg,
|
"""
|
||||||
"image/png" => Philomena.Processors.Png,
|
@spec intensities(map(), String.t()) :: map()
|
||||||
"image/svg+xml" => Philomena.Processors.Svg,
|
def intensities(analysis, file) do
|
||||||
"video/webm" => Philomena.Processors.Webm
|
processor(analysis.mime_type).intensities(analysis, file)
|
||||||
}
|
|
||||||
|> Map.get(type)
|
|
||||||
end
|
|
||||||
|
|
||||||
@versions [
|
|
||||||
thumb_tiny: {50, 50},
|
|
||||||
thumb_small: {150, 150},
|
|
||||||
thumb: {250, 250},
|
|
||||||
small: {320, 240},
|
|
||||||
medium: {800, 600},
|
|
||||||
large: {1280, 1024},
|
|
||||||
tall: {1024, 4096},
|
|
||||||
full: nil
|
|
||||||
]
|
|
||||||
|
|
||||||
def after_upload(image, params) do
|
|
||||||
with upload when not is_nil(upload) <- params["image"],
|
|
||||||
file <- upload.path,
|
|
||||||
{:ok, mime} <- Mime.file(file),
|
|
||||||
mime <- mimes(mime),
|
|
||||||
analyzer when not is_nil(analyzer) <- analyzers(mime),
|
|
||||||
analysis <- analyzer.analyze(file),
|
|
||||||
changes <- analysis_to_changes(analysis, file, upload.filename)
|
|
||||||
do
|
|
||||||
image
|
|
||||||
|> Image.image_changeset(changes)
|
|
||||||
else
|
|
||||||
_ ->
|
|
||||||
image
|
|
||||||
|> Image.image_changeset(%{})
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def after_insert(image) do
|
|
||||||
file = image_file(image)
|
|
||||||
dir = Path.dirname(file)
|
|
||||||
File.mkdir_p!(dir)
|
|
||||||
File.cp!(image.uploaded_image, file)
|
|
||||||
end
|
|
||||||
|
|
||||||
def process_image(image_id) do
|
|
||||||
image = Repo.get!(Image, image_id)
|
|
||||||
|
|
||||||
mime = image.image_mime_type
|
|
||||||
file = image_file(image)
|
|
||||||
analyzer = analyzers(mime)
|
|
||||||
analysis = analyzer.analyze(file)
|
|
||||||
processor = processors(mime)
|
|
||||||
process = processor.process(analysis, file, @versions)
|
|
||||||
|
|
||||||
apply_edit_script(image, process)
|
|
||||||
sha512 = Sha512.file(file)
|
|
||||||
changeset = Image.thumbnail_changeset(image, %{"image_sha512_hash" => sha512})
|
|
||||||
image = Repo.update!(changeset)
|
|
||||||
spawn fn -> DuplicateReports.generate_reports(image) end
|
|
||||||
|
|
||||||
process = processor.post_process(analysis, file)
|
|
||||||
|
|
||||||
apply_edit_script(image, process)
|
|
||||||
sha512 = Sha512.file(file)
|
|
||||||
changeset = Image.process_changeset(image, %{"image_sha512_hash" => sha512})
|
|
||||||
Repo.update!(changeset)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_edit_script(image, changes) do
|
|
||||||
for change <- changes do
|
|
||||||
apply_change(image, change)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_change(image, {:intensities, intensities}) do
|
|
||||||
ImageIntensities.create_image_intensity(image, intensities)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_change(image, {:replace_original, new_file}) do
|
|
||||||
file = image_file(image)
|
|
||||||
|
|
||||||
File.cp(new_file, file)
|
|
||||||
File.chmod(file, 0o755)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_change(image, {:thumbnails, thumbnails}) do
|
|
||||||
thumb_dir = image_thumb_dir(image)
|
|
||||||
|
|
||||||
for thumbnail <- thumbnails do
|
|
||||||
apply_thumbnail(image, thumb_dir, thumbnail)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_thumbnail(_image, thumb_dir, {:copy, new_file, destination}) do
|
|
||||||
new_destination = Path.join([thumb_dir, destination])
|
|
||||||
dir = Path.dirname(new_destination)
|
|
||||||
|
|
||||||
File.mkdir_p!(dir)
|
|
||||||
File.cp!(new_file, new_destination)
|
|
||||||
File.chmod!(new_destination, 0o755)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp apply_thumbnail(image, thumb_dir, {:symlink_original, destination}) do
|
|
||||||
file = Path.absname(image_file(image))
|
|
||||||
new_destination = Path.join([thumb_dir, destination])
|
|
||||||
dir = Path.dirname(new_destination)
|
|
||||||
|
|
||||||
File.mkdir_p!(dir)
|
|
||||||
File.rm(new_destination)
|
|
||||||
platform_symlink(file, new_destination)
|
|
||||||
File.chmod(new_destination, 0o755)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp platform_symlink(source, destination) do
|
|
||||||
case File.ln_s(source, destination) do
|
|
||||||
:ok -> :ok
|
|
||||||
_err -> File.cp!(source, destination)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
defp analysis_to_changes(analysis, file, upload_name) do
|
|
||||||
{width, height} = analysis.dimensions
|
|
||||||
{:ok, %{size: size}} = File.stat(file)
|
|
||||||
sha512 = Sha512.file(file)
|
|
||||||
filename = build_filename(analysis.extension)
|
|
||||||
|
|
||||||
%{
|
|
||||||
"image" => filename,
|
|
||||||
"image_name" => upload_name,
|
|
||||||
"image_width" => width,
|
|
||||||
"image_height" => height,
|
|
||||||
"image_size" => size,
|
|
||||||
"image_format" => analysis.extension,
|
|
||||||
"image_mime_type" => analysis.mime_type,
|
|
||||||
"image_aspect_ratio" => aspect_ratio(width, height),
|
|
||||||
"image_orig_sha512_hash" => sha512,
|
|
||||||
"image_sha512_hash" => sha512,
|
|
||||||
"is_animated" => analysis.animated?,
|
|
||||||
"uploaded_image" => file
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
defp aspect_ratio(_, 0), do: 0.0
|
|
||||||
defp aspect_ratio(w, h), do: w / h
|
|
||||||
|
|
||||||
defp image_file(image) do
|
|
||||||
Path.join([image_file_root(), image.image])
|
|
||||||
end
|
|
||||||
|
|
||||||
defp image_thumb_dir(image) do
|
|
||||||
Path.join([image_thumbnail_root(), time_identifier(image.created_at), to_string(image.id)])
|
|
||||||
end
|
|
||||||
|
|
||||||
defp build_filename(extension) do
|
|
||||||
[
|
|
||||||
time_identifier(DateTime.utc_now()),
|
|
||||||
"/",
|
|
||||||
usec_identifier(),
|
|
||||||
pid_identifier(),
|
|
||||||
".",
|
|
||||||
extension
|
|
||||||
]
|
|
||||||
|> Enum.join()
|
|
||||||
end
|
|
||||||
|
|
||||||
defp time_identifier(time) do
|
|
||||||
Enum.join([time.year, time.month, time.day], "/")
|
|
||||||
end
|
|
||||||
|
|
||||||
defp usec_identifier do
|
|
||||||
DateTime.utc_now()
|
|
||||||
|> DateTime.to_unix(:microsecond)
|
|
||||||
|> to_string()
|
|
||||||
end
|
|
||||||
|
|
||||||
defp pid_identifier do
|
|
||||||
self()
|
|
||||||
|> :erlang.pid_to_list()
|
|
||||||
|> to_string()
|
|
||||||
|> String.replace(~r/[^0-9]/, "")
|
|
||||||
end
|
|
||||||
|
|
||||||
defp image_file_root do
|
|
||||||
Application.get_env(:philomena, :image_file_root)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp image_thumbnail_root do
|
|
||||||
image_file_root() <> "/thumbs"
|
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -32,7 +32,7 @@ defmodule Philomena.Servers.ImageProcessor do
|
||||||
end
|
end
|
||||||
|
|
||||||
defp process(image_id) do
|
defp process(image_id) do
|
||||||
Philomena.Processors.process_image(image_id)
|
Philomena.Images.Thumbnailer.generate_thumbnails(image_id)
|
||||||
rescue
|
rescue
|
||||||
_ ->
|
_ ->
|
||||||
nil
|
nil
|
||||||
|
|
86
lib/philomena/uploader.ex
Normal file
86
lib/philomena/uploader.ex
Normal file
|
@ -0,0 +1,86 @@
|
||||||
|
defmodule Philomena.Uploader do
|
||||||
|
@moduledoc """
|
||||||
|
Upload and processing callback logic for image files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
alias Philomena.Filename
|
||||||
|
alias Philomena.Analyzers
|
||||||
|
alias Philomena.Sha512
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Performs analysis of the passed Plug.Upload, and invokes a changeset
|
||||||
|
callback on the model or changeset passed in with attributes set on
|
||||||
|
the field_name.
|
||||||
|
"""
|
||||||
|
@spec analyze_upload(any(), String.t(), Plug.Upload.t(), (any(), map() -> Ecto.Changeset.t())) :: Ecto.Changeset.t()
|
||||||
|
def analyze_upload(model_or_changeset, field_name, upload_parameter, changeset_fn) do
|
||||||
|
with {:ok, analysis} <- Analyzers.analyze(upload_parameter),
|
||||||
|
analysis <- extra_attributes(analysis, upload_parameter)
|
||||||
|
do
|
||||||
|
attributes =
|
||||||
|
%{
|
||||||
|
"name" => analysis.name,
|
||||||
|
"width" => analysis.width,
|
||||||
|
"height" => analysis.height,
|
||||||
|
"size" => analysis.size,
|
||||||
|
"format" => analysis.extension,
|
||||||
|
"mime_type" => analysis.mime_type,
|
||||||
|
"aspect_ratio" => analysis.aspect_ratio,
|
||||||
|
"orig_sha512_hash" => analysis.sha512,
|
||||||
|
"sha512_hash" => analysis.sha512,
|
||||||
|
"is_animated" => analysis.animated?
|
||||||
|
}
|
||||||
|
|> prefix_attributes(field_name)
|
||||||
|
|> Map.put(field_name, analysis.new_name)
|
||||||
|
|> Map.put(upload_key(field_name), upload_parameter.path)
|
||||||
|
|
||||||
|
changeset_fn.(model_or_changeset, attributes)
|
||||||
|
else
|
||||||
|
_error ->
|
||||||
|
changeset_fn.(model_or_changeset, %{})
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Writes the file to permanent storage. This should be the last step in the
|
||||||
|
transaction.
|
||||||
|
"""
|
||||||
|
@spec persist_upload(any(), String.t(), String.t()) :: any()
|
||||||
|
def persist_upload(model, file_root, field_name) do
|
||||||
|
source = Map.get(model, String.to_existing_atom(upload_key(field_name)))
|
||||||
|
dest = Map.get(model, String.to_existing_atom(field_name))
|
||||||
|
target = Path.join(file_root, dest)
|
||||||
|
dir = Path.dirname(target)
|
||||||
|
|
||||||
|
# Create the target directory if it doesn't exist yet,
|
||||||
|
# then write the file.
|
||||||
|
File.mkdir_p!(dir)
|
||||||
|
File.cp!(source, target)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp extra_attributes(analysis, %Plug.Upload{path: path, filename: filename}) do
|
||||||
|
{width, height} = analysis.dimensions
|
||||||
|
aspect_ratio = aspect_ratio(width, height)
|
||||||
|
|
||||||
|
stat = File.stat!(path)
|
||||||
|
sha512 = Sha512.file(path)
|
||||||
|
new_name = Filename.build(analysis.extension)
|
||||||
|
|
||||||
|
analysis
|
||||||
|
|> Map.put(:size, stat.size)
|
||||||
|
|> Map.put(:name, filename)
|
||||||
|
|> Map.put(:width, width)
|
||||||
|
|> Map.put(:height, height)
|
||||||
|
|> Map.put(:sha512, sha512)
|
||||||
|
|> Map.put(:new_name, new_name)
|
||||||
|
|> Map.put(:aspect_ratio, aspect_ratio)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp aspect_ratio(_, 0), do: 0.0
|
||||||
|
defp aspect_ratio(w, h), do: w / h
|
||||||
|
|
||||||
|
defp prefix_attributes(map, prefix),
|
||||||
|
do: Map.new(map, fn {key, value} -> {"#{prefix}_#{key}", value} end)
|
||||||
|
|
||||||
|
defp upload_key(field_name), do: "uploaded_#{field_name}"
|
||||||
|
end
|
|
@ -1,4 +1,5 @@
|
||||||
defmodule PhilomenaWeb.ImageReverse do
|
defmodule PhilomenaWeb.ImageReverse do
|
||||||
|
alias Philomena.Analyzers
|
||||||
alias Philomena.Processors
|
alias Philomena.Processors
|
||||||
alias Philomena.DuplicateReports
|
alias Philomena.DuplicateReports
|
||||||
alias Philomena.Repo
|
alias Philomena.Repo
|
||||||
|
@ -7,8 +8,6 @@ defmodule PhilomenaWeb.ImageReverse do
|
||||||
def images(image_params) do
|
def images(image_params) do
|
||||||
image_params
|
image_params
|
||||||
|> Map.get("image")
|
|> Map.get("image")
|
||||||
|> Map.get(:path)
|
|
||||||
|> mime()
|
|
||||||
|> analyze()
|
|> analyze()
|
||||||
|> intensities()
|
|> intensities()
|
||||||
|> case do
|
|> case do
|
||||||
|
@ -26,22 +25,14 @@ defmodule PhilomenaWeb.ImageReverse do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp mime(file) do
|
defp analyze(%Plug.Upload{path: path}) do
|
||||||
{:ok, mime} = Philomena.Mime.file(file)
|
{:ok, analysis} = Analyzers.analyze(path)
|
||||||
|
{analysis, path}
|
||||||
{mime, file}
|
|
||||||
end
|
|
||||||
|
|
||||||
defp analyze({mime, file}) do
|
|
||||||
case Processors.analyzers(mime) do
|
|
||||||
nil -> :error
|
|
||||||
a -> {a.analyze(file), mime, file}
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp intensities(:error), do: :error
|
defp intensities(:error), do: :error
|
||||||
defp intensities({analysis, mime, file}) do
|
defp intensities({analysis, path}) do
|
||||||
{analysis, Processors.processors(mime).intensities(analysis, file)}
|
{analysis, Processors.intensities(analysis, path)}
|
||||||
end
|
end
|
||||||
|
|
||||||
# The distance metric is taxicab distance, not Euclidean,
|
# The distance metric is taxicab distance, not Euclidean,
|
||||||
|
|
Loading…
Reference in a new issue