Require a start date for upload_to_s3 task

This commit is contained in:
byte[] 2022-05-14 20:03:33 -04:00
parent 76bf7f292a
commit 07862b0d0c

View file

@ -17,17 +17,37 @@ defmodule Mix.Tasks.UploadToS3 do
@shortdoc "Dumps existing image files to S3 storage backend" @shortdoc "Dumps existing image files to S3 storage backend"
@requirements ["app.start"] @requirements ["app.start"]
@impl Mix.Task @impl Mix.Task
def run([concurrency | args]) do def run(args) do
{concurrency, _} = Integer.parse(concurrency) {args, rest} =
OptionParser.parse_head!(args,
strict: [
concurrency: :integer,
adverts: :boolean,
avatars: :boolean,
badges: :boolean,
tags: :boolean,
images: :boolean
]
)
if Enum.member?(args, "--adverts") do concurrency = Keyword.get(args, :concurrency, 4)
time =
with [time] <- rest,
{:ok, time, _} <- DateTime.from_iso8601(time) do
time
else
_ -> raise ArgumentError, "Must provide a RFC3339 start time, like 1970-01-01T00:00:00Z"
end
if args[:adverts] do
file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts") file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts")
new_file_root = Application.fetch_env!(:philomena, :advert_file_root) new_file_root = Application.fetch_env!(:philomena, :advert_file_root)
IO.puts("\nAdverts:") IO.puts("\nAdverts:")
upload_typical( upload_typical(
where(Advert, [a], not is_nil(a.image)), where(Advert, [a], not is_nil(a.image) and a.updated_at >= ^time),
concurrency, concurrency,
file_root, file_root,
new_file_root, new_file_root,
@ -35,14 +55,14 @@ defmodule Mix.Tasks.UploadToS3 do
) )
end end
if Enum.member?(args, "--avatars") do if args[:avatars] do
file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars") file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars")
new_file_root = Application.fetch_env!(:philomena, :avatar_file_root) new_file_root = Application.fetch_env!(:philomena, :avatar_file_root)
IO.puts("\nAvatars:") IO.puts("\nAvatars:")
upload_typical( upload_typical(
where(User, [u], not is_nil(u.avatar)), where(User, [u], not is_nil(u.avatar) and u.updated_at >= ^time),
concurrency, concurrency,
file_root, file_root,
new_file_root, new_file_root,
@ -50,14 +70,14 @@ defmodule Mix.Tasks.UploadToS3 do
) )
end end
if Enum.member?(args, "--badges") do if args[:badges] do
file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images") file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images")
new_file_root = Application.fetch_env!(:philomena, :badge_file_root) new_file_root = Application.fetch_env!(:philomena, :badge_file_root)
IO.puts("\nBadges:") IO.puts("\nBadges:")
upload_typical( upload_typical(
where(Badge, [b], not is_nil(b.image)), where(Badge, [b], not is_nil(b.image) and b.updated_at >= ^time),
concurrency, concurrency,
file_root, file_root,
new_file_root, new_file_root,
@ -65,14 +85,14 @@ defmodule Mix.Tasks.UploadToS3 do
) )
end end
if Enum.member?(args, "--tags") do if args[:tags] do
file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images") file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images")
new_file_root = Application.fetch_env!(:philomena, :tag_file_root) new_file_root = Application.fetch_env!(:philomena, :tag_file_root)
IO.puts("\nTags:") IO.puts("\nTags:")
upload_typical( upload_typical(
where(Tag, [t], not is_nil(t.image)), where(Tag, [t], not is_nil(t.image) and t.updated_at >= ^time),
concurrency, concurrency,
file_root, file_root,
new_file_root, new_file_root,
@ -80,7 +100,7 @@ defmodule Mix.Tasks.UploadToS3 do
) )
end end
if Enum.member?(args, "--images") do if args[:images] do
file_root = file_root =
Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs") Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs")
@ -90,7 +110,13 @@ defmodule Mix.Tasks.UploadToS3 do
Application.put_env(:philomena, :image_file_root, "") Application.put_env(:philomena, :image_file_root, "")
IO.puts("\nImages:") IO.puts("\nImages:")
upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root)
upload_images(
where(Image, [i], not is_nil(i.image) and i.updated_at >= ^time),
concurrency,
file_root,
new_file_root
)
end end
end end
@ -100,7 +126,7 @@ defmodule Mix.Tasks.UploadToS3 do
|> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name)) |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name))
|> Stream.run() |> Stream.run()
IO.write("\r#{hd(models).id}") IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
end) end)
end end
@ -119,7 +145,7 @@ defmodule Mix.Tasks.UploadToS3 do
|> Task.async_stream(&upload_image_model(&1, file_root, new_file_root)) |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root))
|> Stream.run() |> Stream.run()
IO.write("\r#{hd(models).id}") IO.write("\r#{hd(models).id} (#{DateTime.to_iso8601(hd(models).updated_at)})")
end) end)
end end