Pull out media server and handling into separate container

This commit is contained in:
Liam 2024-12-24 19:16:59 -05:00
parent 0d6bc6cee1
commit 7a54049d42
37 changed files with 1704 additions and 101 deletions

View file

@ -4,3 +4,4 @@ _build
deps
.elixir_ls
priv
native/philomena/target

View file

@ -27,6 +27,7 @@ config :philomena,
image_url_root: System.fetch_env!("IMAGE_URL_ROOT"),
badge_url_root: System.fetch_env!("BADGE_URL_ROOT"),
mailer_address: System.fetch_env!("MAILER_ADDRESS"),
mediaproc_addr: System.fetch_env!("MEDIAPROC_ADDR"),
tag_file_root: System.fetch_env!("TAG_FILE_ROOT"),
site_domains: System.fetch_env!("SITE_DOMAINS"),
tag_url_root: System.fetch_env!("TAG_URL_ROOT"),

View file

@ -30,6 +30,7 @@ services:
- IMAGE_URL_ROOT=/img
- BADGE_URL_ROOT=/badge-img
- TAG_URL_ROOT=/tag-img
- MEDIAPROC_ADDR=mediaproc:1500
- OPENSEARCH_URL=http://opensearch:9200
- REDIS_HOST=valkey
- DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev
@ -52,6 +53,7 @@ services:
- app_deps_data:/srv/philomena/deps
- app_native_data:/srv/philomena/priv/native
depends_on:
- mediaproc
- postgres
- opensearch
- valkey
@ -89,6 +91,18 @@ services:
- .:/srv/philomena
attach: false
mediaproc:
build:
context: .
dockerfile: ./docker/mediaproc/Dockerfile
attach: false
deploy:
resources:
limits:
cpus: '4'
memory: 8gb
pids: 8192
web:
build:
context: .

View file

@ -1,28 +1,12 @@
FROM elixir:1.18.1-alpine
ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json
RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \
&& cp /tmp/repositories /etc/apk/repositories \
&& apk update --allow-untrusted \
&& apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libjpeg-turbo-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg rsvg-convert imagemagick postgresql16-client wget rust cargo --allow-untrusted \
RUN apk add inotify-tools build-base git npm nodejs postgresql16-client wget rust cargo \
&& mix local.hex --force \
&& mix local.rebar --force
ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
RUN git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \
&& cd /tmp/cli_intensities \
&& make -j$(nproc) install
ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
RUN git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \
&& ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \
&& cd /tmp/mediatools \
&& make -j$(nproc) install
COPY docker/app/run-development /usr/local/bin/run-development
COPY docker/app/run-test /usr/local/bin/run-test
COPY docker/app/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert
COPY docker/app/purge-cache /usr/local/bin/purge-cache
ENV PATH=$PATH:/root/.cargo/bin
EXPOSE 5173
CMD run-development
CMD ["/usr/local/bin/run-development"]

View file

@ -0,0 +1,75 @@
FROM rust:1.83-slim
RUN apt update \
&& apt install -y build-essential git libmagic-dev libturbojpeg0-dev libpng-dev \
gifsicle optipng libjpeg-turbo-progs librsvg2-bin librsvg2-dev file imagemagick \
libx264-dev libx265-dev libvpx-dev libdav1d-dev libaom-dev libopus-dev \
libmp3lame-dev libvorbis-dev libwebp-dev libjxl-dev yasm wget
ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/7.1 /tmp/ffmpeg_version.json
ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
RUN wget -qO /tmp/FFmpeg.tar.gz https://github.com/philomena-dev/FFmpeg/archive/refs/heads/release/7.1.tar.gz \
&& wget -qO /tmp/cli_intensities.tar.gz https://github.com/philomena-dev/cli_intensities/archive/refs/heads/master.tar.gz \
&& wget -qO /tmp/mediatools.tar.gz https://github.com/philomena-dev/mediatools/archive/refs/heads/master.tar.gz
RUN cd /tmp \
&& tar -xf FFmpeg.tar.gz \
&& tar -xf cli_intensities.tar.gz \
&& tar -xf mediatools.tar.gz \
&& cd /tmp/FFmpeg-release-7.1 \
&& ./configure \
--prefix=/usr \
--disable-everything \
--disable-stripping \
--disable-static \
--disable-ffplay \
--disable-doc \
--disable-htmlpages \
--disable-manpages \
--disable-podpages \
--disable-txtpages \
--disable-protocols \
--enable-shared \
--enable-pic \
--enable-pthreads \
--enable-gpl \
--enable-avfilter \
--enable-bsf=extract_extradata \
--enable-decoder=aac,apng,av1,gif,h264,hevc,jpeg2000,jpegxl,libaom-av1,libdav1d,libvorbis,libvpx_vp8,libvpx_vp9,mp3,mjpeg,opus,png,vorbis,vp8,vp9,webvtt \
--enable-demuxer=apng,gif,image2,image_gif_pipe,image_jpeg_pipe,image_png_pipe,image_webp_pipe,matroska,mjpeg,mjpeg_2000,mov,webm \
--enable-encoder=aac,apng,gif,jpegxl,libmp3lame,libaom-av1,libvorbis,libopus,libvpx_vp8,libvpx_vp9,libx265,libx264,opus,mjpeg,png,vorbis,webvtt \
--enable-filter=concat,palettegen,paletteuse,scale,setpts,setsar,settb,split,trim \
--enable-libaom \
--enable-libjxl \
--enable-libdav1d \
--enable-libopus \
--enable-libmp3lame \
--enable-libvpx \
--enable-libvorbis \
--enable-libx264 \
--enable-libx265 \
--enable-libwebp \
--enable-muxer=apng,image2,gif,matroska,mp4,webp,webm \
--enable-parser=aac,gif,h264,hevc,jpeg2000,jpegxl,mjpeg,opus,png,vorbis,vp8,vp9,webp \
--enable-protocol=concat,data,file,subfile \
&& make -j$(nproc) install \
&& cd /tmp/cli_intensities-master \
&& make -j$(nproc) install \
&& cd /tmp/mediatools-master \
&& make -j$(nproc) install
COPY native/philomena /tmp/philomena
COPY docker/mediaproc/safe-rsvg-convert /usr/bin/safe-rsvg-convert
RUN cd /tmp/philomena \
&& cargo build --release -p mediaproc_server \
&& cp target/release/mediaproc_server /usr/bin/mediaproc_server
# Set up unprivileged user account
RUN useradd -ms /bin/bash mediaproc
USER mediaproc
WORKDIR /home/mediaproc
ENV RUST_LOG=trace
CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500"]

View file

@ -12,6 +12,10 @@ defmodule Philomena.Native do
@spec camo_image_url(String.t()) :: String.t()
def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
@spec async_process_command(String.t(), String.t(), [String.t()]) :: :ok
def async_process_command(_server_addr, _program, _arguments),
do: :erlang.nif_error(:nif_not_loaded)
@spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()}
def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded)

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
alias PhilomenaMedia.Analyzers.Analyzer
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
@behaviour Analyzer
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
end
defp stats(file) do
case System.cmd("mediastat", [file]) do
case Remote.cmd("mediastat", [file]) do
{output, 0} ->
[_size, frames, width, height, num, den] =
output

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
alias PhilomenaMedia.Analyzers.Analyzer
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
@behaviour Analyzer
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
end
defp stats(file) do
case System.cmd("mediastat", [file]) do
case Remote.cmd("mediastat", [file]) do
{output, 0} ->
[_size, _frames, width, height, num, den] =
output

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
alias PhilomenaMedia.Analyzers.Analyzer
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
@behaviour Analyzer
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
end
defp stats(file) do
case System.cmd("mediastat", [file]) do
case Remote.cmd("mediastat", [file]) do
{output, 0} ->
[_size, frames, width, height, num, den] =
output

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
alias PhilomenaMedia.Analyzers.Analyzer
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
@behaviour Analyzer
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
end
defp stats(file) do
case System.cmd("svgstat", [file]) do
case Remote.cmd("svgstat", [file]) do
{output, 0} ->
[_size, _frames, width, height, _num, _den] =
output

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
alias PhilomenaMedia.Analyzers.Analyzer
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
@behaviour Analyzer
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
end
defp stats(file) do
case System.cmd("mediastat", [file]) do
case Remote.cmd("mediastat", [file]) do
{output, 0} ->
[_size, frames, width, height, num, den] =
output

View file

@ -3,6 +3,8 @@ defmodule PhilomenaMedia.GifPreview do
GIF preview generation for video files.
"""
alias PhilomenaMedia.Remote
@type duration :: float()
@type dimensions :: {pos_integer(), pos_integer()}
@ -49,7 +51,7 @@ defmodule PhilomenaMedia.GifPreview do
end)
{_output, 0} =
System.cmd(
Remote.cmd(
"ffmpeg",
commands(decoder, video, gif, clamp(duration), dimensions, num_images, target_framerate)
)

View file

@ -17,6 +17,8 @@ defmodule PhilomenaMedia.Intensities do
of image dimensions, with poor precision and a poor-to-fair accuracy.
"""
alias PhilomenaMedia.Remote
@type t :: %__MODULE__{
nw: float(),
ne: float(),
@ -50,7 +52,7 @@ defmodule PhilomenaMedia.Intensities do
"""
@spec file(Path.t()) :: {:ok, t()} | :error
def file(input) do
System.cmd("image-intensities", [input])
Remote.cmd("image-intensities", [input])
|> case do
{output, 0} ->
[nw, ne, sw, se] =

View file

@ -27,7 +27,7 @@ defmodule PhilomenaMedia.Mime do
"""
@spec file(Path.t()) :: {:ok, t()} | {:unsupported_mime, t()} | :error
def file(path) do
System.cmd("file", ["-b", "--mime-type", path])
PhilomenaMedia.Remote.cmd("file", ["-b", "--mime-type", path])
|> case do
{output, 0} ->
true_mime(String.trim(output))

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Gif do
alias PhilomenaMedia.Intensities
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
alias PhilomenaMedia.Processors.Processor
alias PhilomenaMedia.Processors
@ -46,7 +47,7 @@ defmodule PhilomenaMedia.Processors.Gif do
defp optimize(file) do
optimized = Briefly.create!(extname: ".gif")
{_output, 0} = System.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
{_output, 0} = Remote.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
optimized
end
@ -54,7 +55,7 @@ defmodule PhilomenaMedia.Processors.Gif do
defp preview(duration, file) do
preview = Briefly.create!(extname: ".png")
{_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview])
{_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview])
preview
end
@ -63,7 +64,7 @@ defmodule PhilomenaMedia.Processors.Gif do
palette = Briefly.create!(extname: ".png")
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Gif do
filter_graph = "[0:v]#{scale_filter}[x];[x][1:v]#{palette_filter}"
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Gif do
mp4 = Briefly.create!(extname: ".mp4")
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -127,7 +128,7 @@ defmodule PhilomenaMedia.Processors.Gif do
])
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
alias PhilomenaMedia.Intensities
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
alias PhilomenaMedia.Processors.Processor
alias PhilomenaMedia.Processors
@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
defp requires_lossy_transformation?(file) do
with {output, 0} <-
System.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]),
Remote.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]),
[orientation, profile] <- String.split(output, "\t") do
orientation not in ["Undefined", "TopLeft"] or profile != ""
else
@ -60,7 +61,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
true ->
# Transcode: strip EXIF, embedded profile and reorient image
{_output, 0} =
System.cmd("convert", [
Remote.cmd("convert", [
file,
"-profile",
srgb_profile(),
@ -71,7 +72,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
_ ->
# Transmux only: Strip EXIF without touching orientation
validate_return(System.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file]))
validate_return(Remote.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file]))
end
stripped
@ -80,7 +81,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
defp optimize(file) do
optimized = Briefly.create!(extname: ".jpg")
validate_return(System.cmd("jpegtran", ["-optimize", "-outfile", optimized, file]))
validate_return(Remote.cmd("jpegtran", ["-optimize", "-outfile", optimized, file]))
optimized
end
@ -90,7 +91,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -103,7 +104,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
scaled
])
{_output, 0} = System.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled])
{_output, 0} = Remote.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled])
[{:copy, scaled, "#{thumb_name}.jpg"}]
end

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Png do
alias PhilomenaMedia.Intensities
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
alias PhilomenaMedia.Processors.Processor
alias PhilomenaMedia.Processors
@ -49,7 +50,7 @@ defmodule PhilomenaMedia.Processors.Png do
optimized = Briefly.create!(extname: ".png")
{_output, 0} =
System.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized])
Remote.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized])
# Remove useless .bak file
File.rm(optimized <> ".bak")
@ -66,7 +67,7 @@ defmodule PhilomenaMedia.Processors.Png do
{_output, 0} =
cond do
animated? ->
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -82,10 +83,10 @@ defmodule PhilomenaMedia.Processors.Png do
])
true ->
System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled])
Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled])
end
System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
[{:copy, scaled, "#{thumb_name}.png"}]
end

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Svg do
alias PhilomenaMedia.Intensities
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
alias PhilomenaMedia.Processors.Processor
alias PhilomenaMedia.Processors
@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Svg do
defp preview(file) do
preview = Briefly.create!(extname: ".png")
{_output, 0} = System.cmd("safe-rsvg-convert", [file, preview])
{_output, 0} = Remote.cmd("safe-rsvg-convert", [file, preview])
preview
end
@ -52,9 +53,9 @@ defmodule PhilomenaMedia.Processors.Svg do
scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
{_output, 0} =
System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled])
Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled])
{_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
{_output, 0} = Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
[{:copy, scaled, "#{thumb_name}.png"}]
end

View file

@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Webm do
alias PhilomenaMedia.Intensities
alias PhilomenaMedia.Analyzers.Result
alias PhilomenaMedia.Remote
alias PhilomenaMedia.GifPreview
alias PhilomenaMedia.Processors.Processor
alias PhilomenaMedia.Processors
@ -56,7 +57,7 @@ defmodule PhilomenaMedia.Processors.Webm do
defp preview(duration, file) do
preview = Briefly.create!(extname: ".png")
{_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview])
{_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview])
preview
end
@ -65,7 +66,7 @@ defmodule PhilomenaMedia.Processors.Webm do
stripped = Briefly.create!(extname: ".webm")
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -110,7 +111,7 @@ defmodule PhilomenaMedia.Processors.Webm do
mp4 = Briefly.create!(extname: ".mp4")
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -170,7 +171,7 @@ defmodule PhilomenaMedia.Processors.Webm do
mp4 = Briefly.create!(extname: ".mp4")
{_output, 0} =
System.cmd("ffmpeg", [
Remote.cmd("ffmpeg", [
"-loglevel",
"0",
"-y",
@ -213,7 +214,7 @@ defmodule PhilomenaMedia.Processors.Webm do
defp select_decoder(file) do
{output, 0} =
System.cmd("ffprobe", [
Remote.cmd("ffprobe", [
"-loglevel",
"0",
"-select_streams",

View file

@ -0,0 +1,18 @@
defmodule PhilomenaMedia.Remote do
@doc """
Out-of-process replacement for `System.cmd/2` that calls the requested
command elsewhere, translating file accesses, and returns the result.
"""
def cmd(command, args) do
:ok = Philomena.Native.async_process_command(mediaproc_addr(), command, args)
receive do
{:command_reply, command_reply} ->
{command_reply.stdout, command_reply.status}
end
end
defp mediaproc_addr do
Application.get_env(:philomena, :mediaproc_addr)
end
end

View file

@ -63,7 +63,7 @@
"redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"},
"remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"},
"req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
"rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"},
"rustler": {:hex, :rustler, "0.35.1", "ec81961ef9ee833d721dafb4449cab29b16b969a3063a842bb9e3ea912f6b938", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "3713b2e70e68ec2bfa8291dfd9cb811fe64a770f254cd9c331f8b34fa7989115"},
"scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"},
"scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]},
"secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"},

File diff suppressed because it is too large Load diff

View file

@ -9,15 +9,25 @@ name = "philomena"
path = "src/lib.rs"
crate-type = ["dylib"]
[workspace]
members = [
"mediaproc",
"mediaproc_client",
"mediaproc_server",
]
default-members = ["mediaproc"]
[dependencies]
base64 = "0.21"
comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false }
http = "0.2"
jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
mediaproc = { path = "./mediaproc" }
once_cell = "1.20"
regex = "1"
ring = "0.16"
rustler = "0.35"
tokio = { version = "1.0", features = ["full"] }
url = "2.5"
zip = { version = "2.2.0", features = ["deflate"], default-features = false }

View file

@ -0,0 +1,10 @@
[package]
name = "mediaproc"
version = "0.1.0"
edition = "2021"
[dependencies]
once_cell = "1.20"
serde = { version = "1.0", features = ["derive"] }
tarpc = { version = "0.35", features = ["full"] }
tokio = { version = "1.0", features = ["full"] }

View file

@ -0,0 +1,146 @@
use std::collections::{HashMap, HashSet};
use std::ffi::OsString;
use std::path::Path;
use std::time::{Duration, Instant};
use crate::{CommandReply, ExecuteCommandError, FileMap, MediaProcessorClient};
use once_cell::sync::Lazy;
use tarpc::context::Context;
#[derive(Default)]
struct CallParameters {
/// Mapping from replaced name to original name.
replacements: HashMap<String, String>,
/// List of post-processed arguments.
arguments: Vec<String>,
/// Mapping of replaced name to file contents.
file_map: FileMap,
}
/// List of file extensions which can be forwarded.
static FORWARDED_EXTS: Lazy<HashSet<OsString>> = Lazy::new(|| {
vec![
"gif", "jpg", "jpeg", "png", "svg", "webm", "webp", "mp4", "icc",
]
.into_iter()
.map(Into::into)
.collect()
});
fn forwarded_ext(path: &Path) -> Option<&str> {
match path.extension() {
Some(ext) if FORWARDED_EXTS.contains(ext) => ext.to_str(),
_ => None,
}
}
fn create_replacements(arguments: impl Iterator<Item = String>) -> CallParameters {
use std::fs::read;
// Maps original name to replaced name.
let mut processed = HashMap::<String, String>::new();
let mut counter: usize = 0;
let mut output = CallParameters::default();
output.arguments = arguments
.map(|arg| {
let path = Path::new(&arg);
// Avoid adding additional replacements if the same file is passed multiple times.
if let Some(replaced_name) = processed.get(&arg) {
return replaced_name.clone();
}
// Only try things that look like paths.
if !path.is_absolute() {
return arg;
}
// Don't forward paths that don't exist or can't be read.
let Ok(contents) = read(path) else {
return arg;
};
// Only forward extension if extension is in allow list.
let replaced_name = match forwarded_ext(path) {
Some(ext) => format!("{}.{}", counter, ext),
None => format!("{}", counter),
};
counter = counter.saturating_add(1);
processed.insert(arg.clone(), replaced_name.clone()); // original -> replaced
output.replacements.insert(replaced_name.clone(), arg); // replaced -> original
output.file_map.insert(replaced_name.clone(), contents); // replaced -> [contents]
replaced_name
})
.collect();
output
}
fn update_replacements(
replacements: HashMap<String, String>,
file_map: FileMap,
) -> Result<(), ExecuteCommandError> {
use std::fs::write;
for (replaced_name, contents) in file_map {
let original_name = replacements
.get(&replaced_name)
.ok_or(ExecuteCommandError::InvalidFileMapName)?;
write(original_name, contents).map_err(|_| ExecuteCommandError::LocalFilesystemError)?;
}
Ok(())
}
fn context_with_1_hour_deadline() -> Context {
let mut context = Context::current();
context.deadline = Instant::now() + Duration::from_secs(60 * 60);
context
}
pub async fn execute_command(
client: &MediaProcessorClient,
program: String,
arguments: Vec<String>,
) -> Result<CommandReply, ExecuteCommandError> {
let call_params = create_replacements(arguments.into_iter());
let (reply, file_map) = client
.execute_command(
context_with_1_hour_deadline(),
program,
call_params.arguments,
call_params.file_map,
)
.await
.map_err(|_| ExecuteCommandError::UnknownError)??;
update_replacements(call_params.replacements, file_map)?;
Ok(reply)
}
pub async fn connect_to_socket_server(server_addr: &str) -> Option<MediaProcessorClient> {
let codec = tarpc::tokio_serde::formats::Bincode::default;
for addr in tokio::net::lookup_host(server_addr).await.ok()? {
let mut transport = tarpc::serde_transport::tcp::connect(addr, codec);
transport.config_mut().max_frame_length(usize::MAX);
let transport = match transport.await {
Ok(transport) => transport,
_ => continue,
};
return Some(
MediaProcessorClient::new(tarpc::client::Config::default(), transport).spawn(),
);
}
None
}

View file

@ -0,0 +1,64 @@
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
pub mod client;
#[tarpc::service]
pub trait MediaProcessor {
/// Executes a command on the media processor server.
async fn execute_command(
program: String,
arguments: Vec<String>,
file_map: FileMap,
) -> Result<(CommandReply, FileMap), ExecuteCommandError>;
}
/// Errors which can occur during command execution.
#[derive(Debug, Deserialize, Serialize)]
pub enum ExecuteCommandError {
/// Requested program was not allowed to be executed.
UnpermittedProgram(String),
/// Failed to launch program.
ExecutionError,
/// File map name character was not allowed ('..', '/', '\\').
InvalidFileMapName,
/// Generic filesystem error.
RemoteFilesystemError,
/// Generic filesystem error.
LocalFilesystemError,
/// Unknown error.
UnknownError,
}
/// Enumeration of permitted program names.
pub static PERMITTED_PROGRAMS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
vec![
"convert",
"ffprobe",
"ffmpeg",
"file",
"gifsicle",
"identify",
"image-intensities",
"jpegtran",
"mediastat",
"mediathumb",
"optipng",
"safe-rsvg-convert",
"svgstat",
]
.into_iter()
.collect()
});
/// Mapping between file name and file contents.
pub type FileMap = HashMap<String, Vec<u8>>;
/// Output reply after command execution has finished.
#[derive(Debug, Deserialize, Serialize)]
pub struct CommandReply {
pub status: u8,
pub stdout: Vec<u8>,
pub stderr: Vec<u8>,
}

View file

@ -0,0 +1,9 @@
[package]
name = "mediaproc_client"
version = "0.1.0"
edition = "2021"
[dependencies]
clap = { version = "4.5", features = ["derive"] }
mediaproc = { path = "../mediaproc" }
tokio = { version = "1.0", features = ["full"] }

View file

@ -0,0 +1,62 @@
use std::io::Write;
use std::process::ExitCode;
use clap::{Parser, Subcommand};
use mediaproc::client::{connect_to_socket_server, execute_command};
use mediaproc::MediaProcessorClient;
#[derive(Parser, Debug)]
#[command(version, about = "RPC Media Processor Client", long_about = None)]
struct Arguments {
/// Server address to connect to, like localhost:1500
server_addr: String,
/// Subcommand to execute.
#[command(subcommand)]
invocation_type: InvocationType,
}
#[derive(Subcommand, Debug)]
enum InvocationType {
/// Execute a command with the given arguments on the remote server.
ExecuteCommand {
/// Program name to execute.
///
/// One of convert, ffprobe, ffmpeg, file, gifsicle, identify,
/// image-intensities, jpegtran, mediastat, optipng, safe-rsvg-convert.
program: String,
/// Arguments to pass to program.
args: Vec<String>,
},
}
#[tokio::main(flavor = "current_thread")]
async fn main() -> ExitCode {
let args = Arguments::parse();
let client = connect_to_socket_server(&args.server_addr)
.await
.expect("failed to connect to server");
match args.invocation_type {
InvocationType::ExecuteCommand { program, args } => {
run_command_client(&client, program, args).await
}
}
}
async fn run_command_client(
client: &MediaProcessorClient,
program: String,
args: Vec<String>,
) -> ExitCode {
let reply = execute_command(client, program, args).await.unwrap();
write_then_drop(std::io::stderr(), reply.stderr);
write_then_drop(std::io::stdout(), reply.stdout);
reply.status.into()
}
fn write_then_drop(mut stream: impl Write, data: Vec<u8>) {
stream.write_all(&data).unwrap()
}

View file

@ -0,0 +1,14 @@
[package]
name = "mediaproc_server"
version = "0.1.0"
edition = "2021"
[dependencies]
env_logger = "0.11"
clap = { version = "4.5", features = ["derive"] }
futures = "0.3"
mediaproc = { path = "../mediaproc" }
tarpc = { version = "0.35", features = ["full"] }
tempfile = "3"
tokio = { version = "1.0", features = ["full"] }
tracing = "0.1"

View file

@ -0,0 +1,63 @@
use std::collections::HashSet;
use std::os::unix::process::ExitStatusExt;
use mediaproc::{CommandReply, ExecuteCommandError, FileMap, PERMITTED_PROGRAMS};
use tokio::process::Command;
fn validate_name(name: &str) -> Result<(), ExecuteCommandError> {
if name == "." || name.contains("..") || name.contains('/') || name.contains('\\') {
return Err(ExecuteCommandError::InvalidFileMapName);
}
Ok(())
}
pub async fn execute_command(
program: String,
arguments: Vec<String>,
file_map: FileMap,
) -> Result<(CommandReply, FileMap), ExecuteCommandError> {
use std::fs::{read, write};
// Check program name.
if !PERMITTED_PROGRAMS.contains(&program.as_ref()) {
return Err(ExecuteCommandError::UnpermittedProgram(program));
}
// Create a new temporary directory which we will work in.
let dir = tempfile::tempdir().map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
// Verify and write out all files.
let mut files = HashSet::<String>::new();
for (name, contents) in file_map {
validate_name(&name)?;
files.insert(name.clone());
let path = dir.path().join(name);
write(path, contents).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
}
// Run the command.
let output = Command::new(program)
.args(arguments)
.current_dir(dir.path())
.output()
.await
.map_err(|_| ExecuteCommandError::ExecutionError)?;
// Read back all files.
let mut file_map = FileMap::new();
for name in files {
let path = dir.path().join(name.clone());
let contents = read(path).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
file_map.insert(name, contents);
}
let reply = CommandReply {
status: output.status.into_raw() as u8,
stdout: output.stdout,
stderr: output.stderr,
};
Ok((reply, file_map))
}

View file

@ -0,0 +1,69 @@
use std::net::SocketAddr;
use clap::Parser;
use futures::{future, Future, StreamExt};
use mediaproc::{CommandReply, ExecuteCommandError, FileMap, MediaProcessor};
use tarpc::context;
use tarpc::server::Channel;
mod command_server;
mod signal;
#[derive(Parser, Debug)]
#[command(version, about = "RPC Media Processor Server", long_about = None)]
struct Arguments {
/// Socket address to bind to, like 127.0.0.1:1500
server_addr: SocketAddr,
}
#[derive(Clone)]
struct MediaProcessorServer;
impl MediaProcessor for MediaProcessorServer {
async fn execute_command(
self,
_: context::Context,
program: String,
arguments: Vec<String>,
file_map: FileMap,
) -> Result<(CommandReply, FileMap), ExecuteCommandError> {
command_server::execute_command(program, arguments, file_map).await
}
}
fn main() {
env_logger::init();
let args = Arguments::parse();
serve(&args);
}
async fn spawn(fut: impl Future<Output = ()> + Send + 'static) {
tokio::spawn(fut);
}
#[tokio::main]
async fn serve(args: &Arguments) {
signal::install_handlers();
let codec = tarpc::tokio_serde::formats::Bincode::default;
let mut listener = tarpc::serde_transport::tcp::listen(args.server_addr, codec)
.await
.unwrap();
listener.config_mut().max_frame_length(usize::MAX);
listener
// Ignore accept errors.
.filter_map(|r| future::ready(r.ok()))
.map(tarpc::server::BaseChannel::with_defaults)
.map(|channel| {
tokio::spawn(
channel
.execute(MediaProcessorServer.serve())
.for_each(spawn),
);
})
.collect()
.await
}

View file

@ -0,0 +1,15 @@
use tokio::signal::unix::{signal, SignalKind};
pub fn install_handlers() {
let mut sigterm = signal(SignalKind::terminate()).unwrap();
let mut sigint = signal(SignalKind::interrupt()).unwrap();
tokio::spawn(async move {
tokio::select! {
_ = sigterm.recv() => tracing::debug!("Received SIGTERM"),
_ = sigint.recv() => tracing::debug!("Received SIGINT"),
};
std::process::exit(1);
});
}

View file

@ -0,0 +1,26 @@
use once_cell::sync::Lazy;
use rustler::{Atom, Env, OwnedEnv, Term};
use std::future::Future;
use std::marker::Send;
use tokio::runtime::Runtime;
static RUNTIME: Lazy<Runtime> = Lazy::new(|| Runtime::new().unwrap());
pub fn call_async<F, T, W>(caller_env: Env, fut: F, w: W) -> Atom
where
F: Future<Output = T> + Send + 'static,
W: for<'a> FnOnce(Env<'a>, T) -> Term<'a>,
W: Send + 'static,
{
let pid = caller_env.pid();
RUNTIME.spawn(async move {
let output = fut.await;
let owned_env = OwnedEnv::new();
owned_env.run(move |env| {
let _ = env.send(&pid, w(env, output));
});
});
rustler::types::atom::ok()
}

View file

@ -1,10 +1,12 @@
use jemallocator::Jemalloc;
use rustler::{Atom, Binary};
use rustler::{Atom, Binary, Env};
use std::collections::HashMap;
mod asyncnif;
mod camo;
mod domains;
mod markdown;
mod remote;
#[cfg(test)]
mod tests;
mod zip;
@ -35,6 +37,19 @@ fn camo_image_url(input: &str) -> String {
camo::image_url_careful(input)
}
// Remote NIF wrappers.
#[rustler::nif]
fn async_process_command(
env: Env,
server_addr: String,
program: String,
arguments: Vec<String>,
) -> Atom {
let fut = remote::process_command(server_addr, program, arguments);
asyncnif::call_async(env, fut, remote::with_env)
}
// Zip NIF wrappers.
#[rustler::nif]

View file

@ -0,0 +1,66 @@
use mediaproc::client::{connect_to_socket_server, execute_command};
use mediaproc::CommandReply;
use rustler::{atoms, Encoder, Env, NifStruct, OwnedBinary, Term};
atoms! {
nil,
command_reply,
}
#[derive(NifStruct)]
#[module = "Elixir.Philomena.Native.CommandReply"]
struct CommandReply_<'a> {
stdout: Term<'a>,
stderr: Term<'a>,
status: u8,
}
fn binary_or_nil<'a>(env: Env<'a>, data: Vec<u8>) -> Term<'a> {
match OwnedBinary::new(data.len()) {
Some(mut binary) => {
binary.copy_from_slice(&data);
binary.release(env).to_term(env)
}
None => nil().to_term(env),
}
}
pub async fn process_command(
server_addr: String,
program: String,
arguments: Vec<String>,
) -> CommandReply {
let client = match connect_to_socket_server(&server_addr).await {
Some(client) => client,
None => {
return CommandReply {
stdout: vec![],
stderr: "failed to connect to server".into(),
status: 255,
}
}
};
match execute_command(&client, program, arguments).await {
Ok(reply) => reply,
Err(err) => CommandReply {
stdout: vec![],
stderr: format!("failed to execute command: {err:?}").into(),
status: 255,
},
}
}
/// Converts the response into a {:command_reply, %CommandReply{...}} message
/// which gets sent back to the caller.
pub fn with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> {
(
command_reply(),
CommandReply_ {
stdout: binary_or_nil(env, r.stdout),
stderr: binary_or_nil(env, r.stderr),
status: r.status,
},
)
.encode(env)
}

View file

@ -46,7 +46,7 @@ request_attributes = [
IO.puts "---- Generating images"
for image_def <- resources["remote_images"] do
file = Briefly.create!()
file = Briefly.create!(extname: ".png")
now = DateTime.utc_now() |> DateTime.to_unix(:microsecond)
IO.puts "Fetching #{image_def["url"]} ..."