mirror of
https://github.com/philomena-dev/philomena.git
synced 2025-01-31 19:36:44 +01:00
Pull out media server and handling into separate container
This commit is contained in:
parent
0d6bc6cee1
commit
7a54049d42
37 changed files with 1704 additions and 101 deletions
|
@ -4,3 +4,4 @@ _build
|
|||
deps
|
||||
.elixir_ls
|
||||
priv
|
||||
native/philomena/target
|
||||
|
|
|
@ -27,6 +27,7 @@ config :philomena,
|
|||
image_url_root: System.fetch_env!("IMAGE_URL_ROOT"),
|
||||
badge_url_root: System.fetch_env!("BADGE_URL_ROOT"),
|
||||
mailer_address: System.fetch_env!("MAILER_ADDRESS"),
|
||||
mediaproc_addr: System.fetch_env!("MEDIAPROC_ADDR"),
|
||||
tag_file_root: System.fetch_env!("TAG_FILE_ROOT"),
|
||||
site_domains: System.fetch_env!("SITE_DOMAINS"),
|
||||
tag_url_root: System.fetch_env!("TAG_URL_ROOT"),
|
||||
|
|
|
@ -30,6 +30,7 @@ services:
|
|||
- IMAGE_URL_ROOT=/img
|
||||
- BADGE_URL_ROOT=/badge-img
|
||||
- TAG_URL_ROOT=/tag-img
|
||||
- MEDIAPROC_ADDR=mediaproc:1500
|
||||
- OPENSEARCH_URL=http://opensearch:9200
|
||||
- REDIS_HOST=valkey
|
||||
- DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev
|
||||
|
@ -52,6 +53,7 @@ services:
|
|||
- app_deps_data:/srv/philomena/deps
|
||||
- app_native_data:/srv/philomena/priv/native
|
||||
depends_on:
|
||||
- mediaproc
|
||||
- postgres
|
||||
- opensearch
|
||||
- valkey
|
||||
|
@ -89,6 +91,18 @@ services:
|
|||
- .:/srv/philomena
|
||||
attach: false
|
||||
|
||||
mediaproc:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./docker/mediaproc/Dockerfile
|
||||
attach: false
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 8gb
|
||||
pids: 8192
|
||||
|
||||
web:
|
||||
build:
|
||||
context: .
|
||||
|
|
|
@ -1,28 +1,12 @@
|
|||
FROM elixir:1.18.1-alpine
|
||||
|
||||
ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json
|
||||
RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \
|
||||
&& cp /tmp/repositories /etc/apk/repositories \
|
||||
&& apk update --allow-untrusted \
|
||||
&& apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libjpeg-turbo-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg rsvg-convert imagemagick postgresql16-client wget rust cargo --allow-untrusted \
|
||||
RUN apk add inotify-tools build-base git npm nodejs postgresql16-client wget rust cargo \
|
||||
&& mix local.hex --force \
|
||||
&& mix local.rebar --force
|
||||
|
||||
ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
|
||||
RUN git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \
|
||||
&& cd /tmp/cli_intensities \
|
||||
&& make -j$(nproc) install
|
||||
|
||||
ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
|
||||
RUN git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \
|
||||
&& ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \
|
||||
&& cd /tmp/mediatools \
|
||||
&& make -j$(nproc) install
|
||||
|
||||
COPY docker/app/run-development /usr/local/bin/run-development
|
||||
COPY docker/app/run-test /usr/local/bin/run-test
|
||||
COPY docker/app/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert
|
||||
COPY docker/app/purge-cache /usr/local/bin/purge-cache
|
||||
ENV PATH=$PATH:/root/.cargo/bin
|
||||
EXPOSE 5173
|
||||
CMD run-development
|
||||
CMD ["/usr/local/bin/run-development"]
|
||||
|
|
75
docker/mediaproc/Dockerfile
Normal file
75
docker/mediaproc/Dockerfile
Normal file
|
@ -0,0 +1,75 @@
|
|||
FROM rust:1.83-slim
|
||||
|
||||
RUN apt update \
|
||||
&& apt install -y build-essential git libmagic-dev libturbojpeg0-dev libpng-dev \
|
||||
gifsicle optipng libjpeg-turbo-progs librsvg2-bin librsvg2-dev file imagemagick \
|
||||
libx264-dev libx265-dev libvpx-dev libdav1d-dev libaom-dev libopus-dev \
|
||||
libmp3lame-dev libvorbis-dev libwebp-dev libjxl-dev yasm wget
|
||||
|
||||
ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/7.1 /tmp/ffmpeg_version.json
|
||||
ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
|
||||
ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
|
||||
|
||||
RUN wget -qO /tmp/FFmpeg.tar.gz https://github.com/philomena-dev/FFmpeg/archive/refs/heads/release/7.1.tar.gz \
|
||||
&& wget -qO /tmp/cli_intensities.tar.gz https://github.com/philomena-dev/cli_intensities/archive/refs/heads/master.tar.gz \
|
||||
&& wget -qO /tmp/mediatools.tar.gz https://github.com/philomena-dev/mediatools/archive/refs/heads/master.tar.gz
|
||||
|
||||
RUN cd /tmp \
|
||||
&& tar -xf FFmpeg.tar.gz \
|
||||
&& tar -xf cli_intensities.tar.gz \
|
||||
&& tar -xf mediatools.tar.gz \
|
||||
&& cd /tmp/FFmpeg-release-7.1 \
|
||||
&& ./configure \
|
||||
--prefix=/usr \
|
||||
--disable-everything \
|
||||
--disable-stripping \
|
||||
--disable-static \
|
||||
--disable-ffplay \
|
||||
--disable-doc \
|
||||
--disable-htmlpages \
|
||||
--disable-manpages \
|
||||
--disable-podpages \
|
||||
--disable-txtpages \
|
||||
--disable-protocols \
|
||||
--enable-shared \
|
||||
--enable-pic \
|
||||
--enable-pthreads \
|
||||
--enable-gpl \
|
||||
--enable-avfilter \
|
||||
--enable-bsf=extract_extradata \
|
||||
--enable-decoder=aac,apng,av1,gif,h264,hevc,jpeg2000,jpegxl,libaom-av1,libdav1d,libvorbis,libvpx_vp8,libvpx_vp9,mp3,mjpeg,opus,png,vorbis,vp8,vp9,webvtt \
|
||||
--enable-demuxer=apng,gif,image2,image_gif_pipe,image_jpeg_pipe,image_png_pipe,image_webp_pipe,matroska,mjpeg,mjpeg_2000,mov,webm \
|
||||
--enable-encoder=aac,apng,gif,jpegxl,libmp3lame,libaom-av1,libvorbis,libopus,libvpx_vp8,libvpx_vp9,libx265,libx264,opus,mjpeg,png,vorbis,webvtt \
|
||||
--enable-filter=concat,palettegen,paletteuse,scale,setpts,setsar,settb,split,trim \
|
||||
--enable-libaom \
|
||||
--enable-libjxl \
|
||||
--enable-libdav1d \
|
||||
--enable-libopus \
|
||||
--enable-libmp3lame \
|
||||
--enable-libvpx \
|
||||
--enable-libvorbis \
|
||||
--enable-libx264 \
|
||||
--enable-libx265 \
|
||||
--enable-libwebp \
|
||||
--enable-muxer=apng,image2,gif,matroska,mp4,webp,webm \
|
||||
--enable-parser=aac,gif,h264,hevc,jpeg2000,jpegxl,mjpeg,opus,png,vorbis,vp8,vp9,webp \
|
||||
--enable-protocol=concat,data,file,subfile \
|
||||
&& make -j$(nproc) install \
|
||||
&& cd /tmp/cli_intensities-master \
|
||||
&& make -j$(nproc) install \
|
||||
&& cd /tmp/mediatools-master \
|
||||
&& make -j$(nproc) install
|
||||
|
||||
COPY native/philomena /tmp/philomena
|
||||
COPY docker/mediaproc/safe-rsvg-convert /usr/bin/safe-rsvg-convert
|
||||
|
||||
RUN cd /tmp/philomena \
|
||||
&& cargo build --release -p mediaproc_server \
|
||||
&& cp target/release/mediaproc_server /usr/bin/mediaproc_server
|
||||
|
||||
# Set up unprivileged user account
|
||||
RUN useradd -ms /bin/bash mediaproc
|
||||
USER mediaproc
|
||||
WORKDIR /home/mediaproc
|
||||
ENV RUST_LOG=trace
|
||||
CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500"]
|
|
@ -12,6 +12,10 @@ defmodule Philomena.Native do
|
|||
@spec camo_image_url(String.t()) :: String.t()
|
||||
def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
|
||||
|
||||
@spec async_process_command(String.t(), String.t(), [String.t()]) :: :ok
|
||||
def async_process_command(_server_addr, _program, _arguments),
|
||||
do: :erlang.nif_error(:nif_not_loaded)
|
||||
|
||||
@spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()}
|
||||
def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded)
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
|
|||
|
||||
alias PhilomenaMedia.Analyzers.Analyzer
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@behaviour Analyzer
|
||||
|
||||
|
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
|
|||
end
|
||||
|
||||
defp stats(file) do
|
||||
case System.cmd("mediastat", [file]) do
|
||||
case Remote.cmd("mediastat", [file]) do
|
||||
{output, 0} ->
|
||||
[_size, frames, width, height, num, den] =
|
||||
output
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
|
|||
|
||||
alias PhilomenaMedia.Analyzers.Analyzer
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@behaviour Analyzer
|
||||
|
||||
|
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
|
|||
end
|
||||
|
||||
defp stats(file) do
|
||||
case System.cmd("mediastat", [file]) do
|
||||
case Remote.cmd("mediastat", [file]) do
|
||||
{output, 0} ->
|
||||
[_size, _frames, width, height, num, den] =
|
||||
output
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
|
|||
|
||||
alias PhilomenaMedia.Analyzers.Analyzer
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@behaviour Analyzer
|
||||
|
||||
|
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
|
|||
end
|
||||
|
||||
defp stats(file) do
|
||||
case System.cmd("mediastat", [file]) do
|
||||
case Remote.cmd("mediastat", [file]) do
|
||||
{output, 0} ->
|
||||
[_size, frames, width, height, num, den] =
|
||||
output
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
|
|||
|
||||
alias PhilomenaMedia.Analyzers.Analyzer
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@behaviour Analyzer
|
||||
|
||||
|
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
|
|||
end
|
||||
|
||||
defp stats(file) do
|
||||
case System.cmd("svgstat", [file]) do
|
||||
case Remote.cmd("svgstat", [file]) do
|
||||
{output, 0} ->
|
||||
[_size, _frames, width, height, _num, _den] =
|
||||
output
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
|
|||
|
||||
alias PhilomenaMedia.Analyzers.Analyzer
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@behaviour Analyzer
|
||||
|
||||
|
@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
|
|||
end
|
||||
|
||||
defp stats(file) do
|
||||
case System.cmd("mediastat", [file]) do
|
||||
case Remote.cmd("mediastat", [file]) do
|
||||
{output, 0} ->
|
||||
[_size, frames, width, height, num, den] =
|
||||
output
|
||||
|
|
|
@ -3,6 +3,8 @@ defmodule PhilomenaMedia.GifPreview do
|
|||
GIF preview generation for video files.
|
||||
"""
|
||||
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@type duration :: float()
|
||||
@type dimensions :: {pos_integer(), pos_integer()}
|
||||
|
||||
|
@ -49,7 +51,7 @@ defmodule PhilomenaMedia.GifPreview do
|
|||
end)
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd(
|
||||
Remote.cmd(
|
||||
"ffmpeg",
|
||||
commands(decoder, video, gif, clamp(duration), dimensions, num_images, target_framerate)
|
||||
)
|
||||
|
|
|
@ -17,6 +17,8 @@ defmodule PhilomenaMedia.Intensities do
|
|||
of image dimensions, with poor precision and a poor-to-fair accuracy.
|
||||
"""
|
||||
|
||||
alias PhilomenaMedia.Remote
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
nw: float(),
|
||||
ne: float(),
|
||||
|
@ -50,7 +52,7 @@ defmodule PhilomenaMedia.Intensities do
|
|||
"""
|
||||
@spec file(Path.t()) :: {:ok, t()} | :error
|
||||
def file(input) do
|
||||
System.cmd("image-intensities", [input])
|
||||
Remote.cmd("image-intensities", [input])
|
||||
|> case do
|
||||
{output, 0} ->
|
||||
[nw, ne, sw, se] =
|
||||
|
|
|
@ -27,7 +27,7 @@ defmodule PhilomenaMedia.Mime do
|
|||
"""
|
||||
@spec file(Path.t()) :: {:ok, t()} | {:unsupported_mime, t()} | :error
|
||||
def file(path) do
|
||||
System.cmd("file", ["-b", "--mime-type", path])
|
||||
PhilomenaMedia.Remote.cmd("file", ["-b", "--mime-type", path])
|
||||
|> case do
|
||||
{output, 0} ->
|
||||
true_mime(String.trim(output))
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
|
||||
alias PhilomenaMedia.Intensities
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
alias PhilomenaMedia.Processors.Processor
|
||||
alias PhilomenaMedia.Processors
|
||||
|
||||
|
@ -46,7 +47,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
defp optimize(file) do
|
||||
optimized = Briefly.create!(extname: ".gif")
|
||||
|
||||
{_output, 0} = System.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
|
||||
{_output, 0} = Remote.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
|
||||
|
||||
optimized
|
||||
end
|
||||
|
@ -54,7 +55,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
defp preview(duration, file) do
|
||||
preview = Briefly.create!(extname: ".png")
|
||||
|
||||
{_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview])
|
||||
{_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview])
|
||||
|
||||
preview
|
||||
end
|
||||
|
@ -63,7 +64,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
palette = Briefly.create!(extname: ".png")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
filter_graph = "[0:v]#{scale_filter}[x];[x][1:v]#{palette_filter}"
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
mp4 = Briefly.create!(extname: ".mp4")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -127,7 +128,7 @@ defmodule PhilomenaMedia.Processors.Gif do
|
|||
])
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
|
||||
alias PhilomenaMedia.Intensities
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
alias PhilomenaMedia.Processors.Processor
|
||||
alias PhilomenaMedia.Processors
|
||||
|
||||
|
@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
|
||||
defp requires_lossy_transformation?(file) do
|
||||
with {output, 0} <-
|
||||
System.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]),
|
||||
Remote.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]),
|
||||
[orientation, profile] <- String.split(output, "\t") do
|
||||
orientation not in ["Undefined", "TopLeft"] or profile != ""
|
||||
else
|
||||
|
@ -60,7 +61,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
true ->
|
||||
# Transcode: strip EXIF, embedded profile and reorient image
|
||||
{_output, 0} =
|
||||
System.cmd("convert", [
|
||||
Remote.cmd("convert", [
|
||||
file,
|
||||
"-profile",
|
||||
srgb_profile(),
|
||||
|
@ -71,7 +72,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
|
||||
_ ->
|
||||
# Transmux only: Strip EXIF without touching orientation
|
||||
validate_return(System.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file]))
|
||||
validate_return(Remote.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file]))
|
||||
end
|
||||
|
||||
stripped
|
||||
|
@ -80,7 +81,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
defp optimize(file) do
|
||||
optimized = Briefly.create!(extname: ".jpg")
|
||||
|
||||
validate_return(System.cmd("jpegtran", ["-optimize", "-outfile", optimized, file]))
|
||||
validate_return(Remote.cmd("jpegtran", ["-optimize", "-outfile", optimized, file]))
|
||||
|
||||
optimized
|
||||
end
|
||||
|
@ -90,7 +91,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -103,7 +104,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do
|
|||
scaled
|
||||
])
|
||||
|
||||
{_output, 0} = System.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled])
|
||||
{_output, 0} = Remote.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled])
|
||||
|
||||
[{:copy, scaled, "#{thumb_name}.jpg"}]
|
||||
end
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Png do
|
|||
|
||||
alias PhilomenaMedia.Intensities
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
alias PhilomenaMedia.Processors.Processor
|
||||
alias PhilomenaMedia.Processors
|
||||
|
||||
|
@ -49,7 +50,7 @@ defmodule PhilomenaMedia.Processors.Png do
|
|||
optimized = Briefly.create!(extname: ".png")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized])
|
||||
Remote.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized])
|
||||
|
||||
# Remove useless .bak file
|
||||
File.rm(optimized <> ".bak")
|
||||
|
@ -66,7 +67,7 @@ defmodule PhilomenaMedia.Processors.Png do
|
|||
{_output, 0} =
|
||||
cond do
|
||||
animated? ->
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -82,10 +83,10 @@ defmodule PhilomenaMedia.Processors.Png do
|
|||
])
|
||||
|
||||
true ->
|
||||
System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled])
|
||||
Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled])
|
||||
end
|
||||
|
||||
System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
|
||||
Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
|
||||
|
||||
[{:copy, scaled, "#{thumb_name}.png"}]
|
||||
end
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Svg do
|
|||
|
||||
alias PhilomenaMedia.Intensities
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
alias PhilomenaMedia.Processors.Processor
|
||||
alias PhilomenaMedia.Processors
|
||||
|
||||
|
@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Svg do
|
|||
defp preview(file) do
|
||||
preview = Briefly.create!(extname: ".png")
|
||||
|
||||
{_output, 0} = System.cmd("safe-rsvg-convert", [file, preview])
|
||||
{_output, 0} = Remote.cmd("safe-rsvg-convert", [file, preview])
|
||||
|
||||
preview
|
||||
end
|
||||
|
@ -52,9 +53,9 @@ defmodule PhilomenaMedia.Processors.Svg do
|
|||
scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease"
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled])
|
||||
Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled])
|
||||
|
||||
{_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
|
||||
{_output, 0} = Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled])
|
||||
|
||||
[{:copy, scaled, "#{thumb_name}.png"}]
|
||||
end
|
||||
|
|
|
@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
|
||||
alias PhilomenaMedia.Intensities
|
||||
alias PhilomenaMedia.Analyzers.Result
|
||||
alias PhilomenaMedia.Remote
|
||||
alias PhilomenaMedia.GifPreview
|
||||
alias PhilomenaMedia.Processors.Processor
|
||||
alias PhilomenaMedia.Processors
|
||||
|
@ -56,7 +57,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
defp preview(duration, file) do
|
||||
preview = Briefly.create!(extname: ".png")
|
||||
|
||||
{_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview])
|
||||
{_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview])
|
||||
|
||||
preview
|
||||
end
|
||||
|
@ -65,7 +66,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
stripped = Briefly.create!(extname: ".webm")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -110,7 +111,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
mp4 = Briefly.create!(extname: ".mp4")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -170,7 +171,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
mp4 = Briefly.create!(extname: ".mp4")
|
||||
|
||||
{_output, 0} =
|
||||
System.cmd("ffmpeg", [
|
||||
Remote.cmd("ffmpeg", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-y",
|
||||
|
@ -213,7 +214,7 @@ defmodule PhilomenaMedia.Processors.Webm do
|
|||
|
||||
defp select_decoder(file) do
|
||||
{output, 0} =
|
||||
System.cmd("ffprobe", [
|
||||
Remote.cmd("ffprobe", [
|
||||
"-loglevel",
|
||||
"0",
|
||||
"-select_streams",
|
||||
|
|
18
lib/philomena_media/remote.ex
Normal file
18
lib/philomena_media/remote.ex
Normal file
|
@ -0,0 +1,18 @@
|
|||
defmodule PhilomenaMedia.Remote do
|
||||
@doc """
|
||||
Out-of-process replacement for `System.cmd/2` that calls the requested
|
||||
command elsewhere, translating file accesses, and returns the result.
|
||||
"""
|
||||
def cmd(command, args) do
|
||||
:ok = Philomena.Native.async_process_command(mediaproc_addr(), command, args)
|
||||
|
||||
receive do
|
||||
{:command_reply, command_reply} ->
|
||||
{command_reply.stdout, command_reply.status}
|
||||
end
|
||||
end
|
||||
|
||||
defp mediaproc_addr do
|
||||
Application.get_env(:philomena, :mediaproc_addr)
|
||||
end
|
||||
end
|
2
mix.lock
2
mix.lock
|
@ -63,7 +63,7 @@
|
|||
"redix": {:hex, :redix, "1.5.1", "a2386971e69bf23630fb3a215a831b5478d2ee7dc9ea7ac811ed89186ab5d7b7", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "85224eb2b683c516b80d472eb89b76067d5866913bf0be59d646f550de71f5c4"},
|
||||
"remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"},
|
||||
"req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
|
||||
"rustler": {:hex, :rustler, "0.35.0", "1e2e379e1150fab9982454973c74ac9899bd0377b3882166ee04127ea613b2d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "a176bea1bb6711474f9dfad282066f2b7392e246459bf4e29dfff6d828779fdf"},
|
||||
"rustler": {:hex, :rustler, "0.35.1", "ec81961ef9ee833d721dafb4449cab29b16b969a3063a842bb9e3ea912f6b938", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "3713b2e70e68ec2bfa8291dfd9cb811fe64a770f254cd9c331f8b34fa7989115"},
|
||||
"scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"},
|
||||
"scrivener_ecto": {:git, "https://github.com/krns/scrivener_ecto.git", "eaad1ddd86a9c8ffa422479417221265a0673777", [ref: "eaad1ddd86a9c8ffa422479417221265a0673777"]},
|
||||
"secure_compare": {:hex, :secure_compare, "0.1.0", "01b3c93c8edb696e8a5b38397ed48e10958c8a5ec740606656445bcbec0aadb8", [:mix], [], "hexpm", "6391a49eb4a6182f0d7425842fc774bbed715e78b2bfb0c83b99c94e02c78b5c"},
|
||||
|
|
1019
native/philomena/Cargo.lock
generated
1019
native/philomena/Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
@ -9,15 +9,25 @@ name = "philomena"
|
|||
path = "src/lib.rs"
|
||||
crate-type = ["dylib"]
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
"mediaproc",
|
||||
"mediaproc_client",
|
||||
"mediaproc_server",
|
||||
]
|
||||
default-members = ["mediaproc"]
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.21"
|
||||
comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.29.2", default-features = false }
|
||||
http = "0.2"
|
||||
jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }
|
||||
mediaproc = { path = "./mediaproc" }
|
||||
once_cell = "1.20"
|
||||
regex = "1"
|
||||
ring = "0.16"
|
||||
rustler = "0.35"
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
url = "2.5"
|
||||
zip = { version = "2.2.0", features = ["deflate"], default-features = false }
|
||||
|
||||
|
|
10
native/philomena/mediaproc/Cargo.toml
Normal file
10
native/philomena/mediaproc/Cargo.toml
Normal file
|
@ -0,0 +1,10 @@
|
|||
[package]
|
||||
name = "mediaproc"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
once_cell = "1.20"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
tarpc = { version = "0.35", features = ["full"] }
|
||||
tokio = { version = "1.0", features = ["full"] }
|
146
native/philomena/mediaproc/src/client.rs
Normal file
146
native/philomena/mediaproc/src/client.rs
Normal file
|
@ -0,0 +1,146 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::ffi::OsString;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::{CommandReply, ExecuteCommandError, FileMap, MediaProcessorClient};
|
||||
use once_cell::sync::Lazy;
|
||||
use tarpc::context::Context;
|
||||
|
||||
#[derive(Default)]
|
||||
struct CallParameters {
|
||||
/// Mapping from replaced name to original name.
|
||||
replacements: HashMap<String, String>,
|
||||
/// List of post-processed arguments.
|
||||
arguments: Vec<String>,
|
||||
/// Mapping of replaced name to file contents.
|
||||
file_map: FileMap,
|
||||
}
|
||||
|
||||
/// List of file extensions which can be forwarded.
|
||||
static FORWARDED_EXTS: Lazy<HashSet<OsString>> = Lazy::new(|| {
|
||||
vec![
|
||||
"gif", "jpg", "jpeg", "png", "svg", "webm", "webp", "mp4", "icc",
|
||||
]
|
||||
.into_iter()
|
||||
.map(Into::into)
|
||||
.collect()
|
||||
});
|
||||
|
||||
fn forwarded_ext(path: &Path) -> Option<&str> {
|
||||
match path.extension() {
|
||||
Some(ext) if FORWARDED_EXTS.contains(ext) => ext.to_str(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_replacements(arguments: impl Iterator<Item = String>) -> CallParameters {
|
||||
use std::fs::read;
|
||||
|
||||
// Maps original name to replaced name.
|
||||
let mut processed = HashMap::<String, String>::new();
|
||||
let mut counter: usize = 0;
|
||||
|
||||
let mut output = CallParameters::default();
|
||||
|
||||
output.arguments = arguments
|
||||
.map(|arg| {
|
||||
let path = Path::new(&arg);
|
||||
|
||||
// Avoid adding additional replacements if the same file is passed multiple times.
|
||||
if let Some(replaced_name) = processed.get(&arg) {
|
||||
return replaced_name.clone();
|
||||
}
|
||||
|
||||
// Only try things that look like paths.
|
||||
if !path.is_absolute() {
|
||||
return arg;
|
||||
}
|
||||
|
||||
// Don't forward paths that don't exist or can't be read.
|
||||
let Ok(contents) = read(path) else {
|
||||
return arg;
|
||||
};
|
||||
|
||||
// Only forward extension if extension is in allow list.
|
||||
let replaced_name = match forwarded_ext(path) {
|
||||
Some(ext) => format!("{}.{}", counter, ext),
|
||||
None => format!("{}", counter),
|
||||
};
|
||||
|
||||
counter = counter.saturating_add(1);
|
||||
|
||||
processed.insert(arg.clone(), replaced_name.clone()); // original -> replaced
|
||||
output.replacements.insert(replaced_name.clone(), arg); // replaced -> original
|
||||
output.file_map.insert(replaced_name.clone(), contents); // replaced -> [contents]
|
||||
|
||||
replaced_name
|
||||
})
|
||||
.collect();
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
fn update_replacements(
|
||||
replacements: HashMap<String, String>,
|
||||
file_map: FileMap,
|
||||
) -> Result<(), ExecuteCommandError> {
|
||||
use std::fs::write;
|
||||
|
||||
for (replaced_name, contents) in file_map {
|
||||
let original_name = replacements
|
||||
.get(&replaced_name)
|
||||
.ok_or(ExecuteCommandError::InvalidFileMapName)?;
|
||||
|
||||
write(original_name, contents).map_err(|_| ExecuteCommandError::LocalFilesystemError)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn context_with_1_hour_deadline() -> Context {
|
||||
let mut context = Context::current();
|
||||
context.deadline = Instant::now() + Duration::from_secs(60 * 60);
|
||||
context
|
||||
}
|
||||
|
||||
pub async fn execute_command(
|
||||
client: &MediaProcessorClient,
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
) -> Result<CommandReply, ExecuteCommandError> {
|
||||
let call_params = create_replacements(arguments.into_iter());
|
||||
let (reply, file_map) = client
|
||||
.execute_command(
|
||||
context_with_1_hour_deadline(),
|
||||
program,
|
||||
call_params.arguments,
|
||||
call_params.file_map,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| ExecuteCommandError::UnknownError)??;
|
||||
|
||||
update_replacements(call_params.replacements, file_map)?;
|
||||
|
||||
Ok(reply)
|
||||
}
|
||||
|
||||
pub async fn connect_to_socket_server(server_addr: &str) -> Option<MediaProcessorClient> {
|
||||
let codec = tarpc::tokio_serde::formats::Bincode::default;
|
||||
|
||||
for addr in tokio::net::lookup_host(server_addr).await.ok()? {
|
||||
let mut transport = tarpc::serde_transport::tcp::connect(addr, codec);
|
||||
transport.config_mut().max_frame_length(usize::MAX);
|
||||
|
||||
let transport = match transport.await {
|
||||
Ok(transport) => transport,
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
return Some(
|
||||
MediaProcessorClient::new(tarpc::client::Config::default(), transport).spawn(),
|
||||
);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
64
native/philomena/mediaproc/src/lib.rs
Normal file
64
native/philomena/mediaproc/src/lib.rs
Normal file
|
@ -0,0 +1,64 @@
|
|||
use once_cell::sync::Lazy;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
pub mod client;
|
||||
|
||||
#[tarpc::service]
|
||||
pub trait MediaProcessor {
|
||||
/// Executes a command on the media processor server.
|
||||
async fn execute_command(
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
file_map: FileMap,
|
||||
) -> Result<(CommandReply, FileMap), ExecuteCommandError>;
|
||||
}
|
||||
|
||||
/// Errors which can occur during command execution.
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub enum ExecuteCommandError {
|
||||
/// Requested program was not allowed to be executed.
|
||||
UnpermittedProgram(String),
|
||||
/// Failed to launch program.
|
||||
ExecutionError,
|
||||
/// File map name character was not allowed ('..', '/', '\\').
|
||||
InvalidFileMapName,
|
||||
/// Generic filesystem error.
|
||||
RemoteFilesystemError,
|
||||
/// Generic filesystem error.
|
||||
LocalFilesystemError,
|
||||
/// Unknown error.
|
||||
UnknownError,
|
||||
}
|
||||
|
||||
/// Enumeration of permitted program names.
|
||||
pub static PERMITTED_PROGRAMS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
|
||||
vec![
|
||||
"convert",
|
||||
"ffprobe",
|
||||
"ffmpeg",
|
||||
"file",
|
||||
"gifsicle",
|
||||
"identify",
|
||||
"image-intensities",
|
||||
"jpegtran",
|
||||
"mediastat",
|
||||
"mediathumb",
|
||||
"optipng",
|
||||
"safe-rsvg-convert",
|
||||
"svgstat",
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
});
|
||||
|
||||
/// Mapping between file name and file contents.
|
||||
pub type FileMap = HashMap<String, Vec<u8>>;
|
||||
|
||||
/// Output reply after command execution has finished.
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct CommandReply {
|
||||
pub status: u8,
|
||||
pub stdout: Vec<u8>,
|
||||
pub stderr: Vec<u8>,
|
||||
}
|
9
native/philomena/mediaproc_client/Cargo.toml
Normal file
9
native/philomena/mediaproc_client/Cargo.toml
Normal file
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "mediaproc_client"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
mediaproc = { path = "../mediaproc" }
|
||||
tokio = { version = "1.0", features = ["full"] }
|
62
native/philomena/mediaproc_client/src/main.rs
Normal file
62
native/philomena/mediaproc_client/src/main.rs
Normal file
|
@ -0,0 +1,62 @@
|
|||
use std::io::Write;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use mediaproc::client::{connect_to_socket_server, execute_command};
|
||||
use mediaproc::MediaProcessorClient;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about = "RPC Media Processor Client", long_about = None)]
|
||||
struct Arguments {
|
||||
/// Server address to connect to, like localhost:1500
|
||||
server_addr: String,
|
||||
|
||||
/// Subcommand to execute.
|
||||
#[command(subcommand)]
|
||||
invocation_type: InvocationType,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum InvocationType {
|
||||
/// Execute a command with the given arguments on the remote server.
|
||||
ExecuteCommand {
|
||||
/// Program name to execute.
|
||||
///
|
||||
/// One of convert, ffprobe, ffmpeg, file, gifsicle, identify,
|
||||
/// image-intensities, jpegtran, mediastat, optipng, safe-rsvg-convert.
|
||||
program: String,
|
||||
/// Arguments to pass to program.
|
||||
args: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> ExitCode {
|
||||
let args = Arguments::parse();
|
||||
let client = connect_to_socket_server(&args.server_addr)
|
||||
.await
|
||||
.expect("failed to connect to server");
|
||||
|
||||
match args.invocation_type {
|
||||
InvocationType::ExecuteCommand { program, args } => {
|
||||
run_command_client(&client, program, args).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_command_client(
|
||||
client: &MediaProcessorClient,
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
) -> ExitCode {
|
||||
let reply = execute_command(client, program, args).await.unwrap();
|
||||
|
||||
write_then_drop(std::io::stderr(), reply.stderr);
|
||||
write_then_drop(std::io::stdout(), reply.stdout);
|
||||
|
||||
reply.status.into()
|
||||
}
|
||||
|
||||
fn write_then_drop(mut stream: impl Write, data: Vec<u8>) {
|
||||
stream.write_all(&data).unwrap()
|
||||
}
|
14
native/philomena/mediaproc_server/Cargo.toml
Normal file
14
native/philomena/mediaproc_server/Cargo.toml
Normal file
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "mediaproc_server"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
env_logger = "0.11"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
futures = "0.3"
|
||||
mediaproc = { path = "../mediaproc" }
|
||||
tarpc = { version = "0.35", features = ["full"] }
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
tracing = "0.1"
|
63
native/philomena/mediaproc_server/src/command_server.rs
Normal file
63
native/philomena/mediaproc_server/src/command_server.rs
Normal file
|
@ -0,0 +1,63 @@
|
|||
use std::collections::HashSet;
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
|
||||
use mediaproc::{CommandReply, ExecuteCommandError, FileMap, PERMITTED_PROGRAMS};
|
||||
use tokio::process::Command;
|
||||
|
||||
fn validate_name(name: &str) -> Result<(), ExecuteCommandError> {
|
||||
if name == "." || name.contains("..") || name.contains('/') || name.contains('\\') {
|
||||
return Err(ExecuteCommandError::InvalidFileMapName);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn execute_command(
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
file_map: FileMap,
|
||||
) -> Result<(CommandReply, FileMap), ExecuteCommandError> {
|
||||
use std::fs::{read, write};
|
||||
|
||||
// Check program name.
|
||||
if !PERMITTED_PROGRAMS.contains(&program.as_ref()) {
|
||||
return Err(ExecuteCommandError::UnpermittedProgram(program));
|
||||
}
|
||||
|
||||
// Create a new temporary directory which we will work in.
|
||||
let dir = tempfile::tempdir().map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
|
||||
|
||||
// Verify and write out all files.
|
||||
let mut files = HashSet::<String>::new();
|
||||
for (name, contents) in file_map {
|
||||
validate_name(&name)?;
|
||||
files.insert(name.clone());
|
||||
|
||||
let path = dir.path().join(name);
|
||||
write(path, contents).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
|
||||
}
|
||||
|
||||
// Run the command.
|
||||
let output = Command::new(program)
|
||||
.args(arguments)
|
||||
.current_dir(dir.path())
|
||||
.output()
|
||||
.await
|
||||
.map_err(|_| ExecuteCommandError::ExecutionError)?;
|
||||
|
||||
// Read back all files.
|
||||
let mut file_map = FileMap::new();
|
||||
for name in files {
|
||||
let path = dir.path().join(name.clone());
|
||||
let contents = read(path).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?;
|
||||
file_map.insert(name, contents);
|
||||
}
|
||||
|
||||
let reply = CommandReply {
|
||||
status: output.status.into_raw() as u8,
|
||||
stdout: output.stdout,
|
||||
stderr: output.stderr,
|
||||
};
|
||||
|
||||
Ok((reply, file_map))
|
||||
}
|
69
native/philomena/mediaproc_server/src/main.rs
Normal file
69
native/philomena/mediaproc_server/src/main.rs
Normal file
|
@ -0,0 +1,69 @@
|
|||
use std::net::SocketAddr;
|
||||
|
||||
use clap::Parser;
|
||||
use futures::{future, Future, StreamExt};
|
||||
use mediaproc::{CommandReply, ExecuteCommandError, FileMap, MediaProcessor};
|
||||
use tarpc::context;
|
||||
use tarpc::server::Channel;
|
||||
|
||||
mod command_server;
|
||||
mod signal;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about = "RPC Media Processor Server", long_about = None)]
|
||||
struct Arguments {
|
||||
/// Socket address to bind to, like 127.0.0.1:1500
|
||||
server_addr: SocketAddr,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MediaProcessorServer;
|
||||
|
||||
impl MediaProcessor for MediaProcessorServer {
|
||||
async fn execute_command(
|
||||
self,
|
||||
_: context::Context,
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
file_map: FileMap,
|
||||
) -> Result<(CommandReply, FileMap), ExecuteCommandError> {
|
||||
command_server::execute_command(program, arguments, file_map).await
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
|
||||
let args = Arguments::parse();
|
||||
|
||||
serve(&args);
|
||||
}
|
||||
|
||||
async fn spawn(fut: impl Future<Output = ()> + Send + 'static) {
|
||||
tokio::spawn(fut);
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn serve(args: &Arguments) {
|
||||
signal::install_handlers();
|
||||
|
||||
let codec = tarpc::tokio_serde::formats::Bincode::default;
|
||||
let mut listener = tarpc::serde_transport::tcp::listen(args.server_addr, codec)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
listener.config_mut().max_frame_length(usize::MAX);
|
||||
listener
|
||||
// Ignore accept errors.
|
||||
.filter_map(|r| future::ready(r.ok()))
|
||||
.map(tarpc::server::BaseChannel::with_defaults)
|
||||
.map(|channel| {
|
||||
tokio::spawn(
|
||||
channel
|
||||
.execute(MediaProcessorServer.serve())
|
||||
.for_each(spawn),
|
||||
);
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
}
|
15
native/philomena/mediaproc_server/src/signal.rs
Normal file
15
native/philomena/mediaproc_server/src/signal.rs
Normal file
|
@ -0,0 +1,15 @@
|
|||
use tokio::signal::unix::{signal, SignalKind};
|
||||
|
||||
pub fn install_handlers() {
|
||||
let mut sigterm = signal(SignalKind::terminate()).unwrap();
|
||||
let mut sigint = signal(SignalKind::interrupt()).unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = sigterm.recv() => tracing::debug!("Received SIGTERM"),
|
||||
_ = sigint.recv() => tracing::debug!("Received SIGINT"),
|
||||
};
|
||||
|
||||
std::process::exit(1);
|
||||
});
|
||||
}
|
26
native/philomena/src/asyncnif.rs
Normal file
26
native/philomena/src/asyncnif.rs
Normal file
|
@ -0,0 +1,26 @@
|
|||
use once_cell::sync::Lazy;
|
||||
use rustler::{Atom, Env, OwnedEnv, Term};
|
||||
use std::future::Future;
|
||||
use std::marker::Send;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
static RUNTIME: Lazy<Runtime> = Lazy::new(|| Runtime::new().unwrap());
|
||||
|
||||
pub fn call_async<F, T, W>(caller_env: Env, fut: F, w: W) -> Atom
|
||||
where
|
||||
F: Future<Output = T> + Send + 'static,
|
||||
W: for<'a> FnOnce(Env<'a>, T) -> Term<'a>,
|
||||
W: Send + 'static,
|
||||
{
|
||||
let pid = caller_env.pid();
|
||||
|
||||
RUNTIME.spawn(async move {
|
||||
let output = fut.await;
|
||||
let owned_env = OwnedEnv::new();
|
||||
owned_env.run(move |env| {
|
||||
let _ = env.send(&pid, w(env, output));
|
||||
});
|
||||
});
|
||||
|
||||
rustler::types::atom::ok()
|
||||
}
|
|
@ -1,10 +1,12 @@
|
|||
use jemallocator::Jemalloc;
|
||||
use rustler::{Atom, Binary};
|
||||
use rustler::{Atom, Binary, Env};
|
||||
use std::collections::HashMap;
|
||||
|
||||
mod asyncnif;
|
||||
mod camo;
|
||||
mod domains;
|
||||
mod markdown;
|
||||
mod remote;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod zip;
|
||||
|
@ -35,6 +37,19 @@ fn camo_image_url(input: &str) -> String {
|
|||
camo::image_url_careful(input)
|
||||
}
|
||||
|
||||
// Remote NIF wrappers.
|
||||
|
||||
#[rustler::nif]
|
||||
fn async_process_command(
|
||||
env: Env,
|
||||
server_addr: String,
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
) -> Atom {
|
||||
let fut = remote::process_command(server_addr, program, arguments);
|
||||
asyncnif::call_async(env, fut, remote::with_env)
|
||||
}
|
||||
|
||||
// Zip NIF wrappers.
|
||||
|
||||
#[rustler::nif]
|
||||
|
|
66
native/philomena/src/remote.rs
Normal file
66
native/philomena/src/remote.rs
Normal file
|
@ -0,0 +1,66 @@
|
|||
use mediaproc::client::{connect_to_socket_server, execute_command};
|
||||
use mediaproc::CommandReply;
|
||||
use rustler::{atoms, Encoder, Env, NifStruct, OwnedBinary, Term};
|
||||
|
||||
atoms! {
|
||||
nil,
|
||||
command_reply,
|
||||
}
|
||||
|
||||
#[derive(NifStruct)]
|
||||
#[module = "Elixir.Philomena.Native.CommandReply"]
|
||||
struct CommandReply_<'a> {
|
||||
stdout: Term<'a>,
|
||||
stderr: Term<'a>,
|
||||
status: u8,
|
||||
}
|
||||
|
||||
fn binary_or_nil<'a>(env: Env<'a>, data: Vec<u8>) -> Term<'a> {
|
||||
match OwnedBinary::new(data.len()) {
|
||||
Some(mut binary) => {
|
||||
binary.copy_from_slice(&data);
|
||||
binary.release(env).to_term(env)
|
||||
}
|
||||
None => nil().to_term(env),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn process_command(
|
||||
server_addr: String,
|
||||
program: String,
|
||||
arguments: Vec<String>,
|
||||
) -> CommandReply {
|
||||
let client = match connect_to_socket_server(&server_addr).await {
|
||||
Some(client) => client,
|
||||
None => {
|
||||
return CommandReply {
|
||||
stdout: vec![],
|
||||
stderr: "failed to connect to server".into(),
|
||||
status: 255,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match execute_command(&client, program, arguments).await {
|
||||
Ok(reply) => reply,
|
||||
Err(err) => CommandReply {
|
||||
stdout: vec![],
|
||||
stderr: format!("failed to execute command: {err:?}").into(),
|
||||
status: 255,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the response into a {:command_reply, %CommandReply{...}} message
|
||||
/// which gets sent back to the caller.
|
||||
pub fn with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> {
|
||||
(
|
||||
command_reply(),
|
||||
CommandReply_ {
|
||||
stdout: binary_or_nil(env, r.stdout),
|
||||
stderr: binary_or_nil(env, r.stderr),
|
||||
status: r.status,
|
||||
},
|
||||
)
|
||||
.encode(env)
|
||||
}
|
|
@ -46,7 +46,7 @@ request_attributes = [
|
|||
|
||||
IO.puts "---- Generating images"
|
||||
for image_def <- resources["remote_images"] do
|
||||
file = Briefly.create!()
|
||||
file = Briefly.create!(extname: ".png")
|
||||
now = DateTime.utc_now() |> DateTime.to_unix(:microsecond)
|
||||
|
||||
IO.puts "Fetching #{image_def["url"]} ..."
|
||||
|
|
Loading…
Reference in a new issue