diff --git a/docker/app/run-development b/docker/app/run-development index 3fce8e72..4e03fab9 100755 --- a/docker/app/run-development +++ b/docker/app/run-development @@ -43,11 +43,6 @@ until wget -qO - elasticsearch:9200; do sleep 2 done -until wget -qO /dev/null --spider files/philomena; do - echo -n "." - sleep 2 -done - echo # Try to create the database if it doesn't exist yet diff --git a/lib/mix/tasks/upload_to_s3.ex b/lib/mix/tasks/upload_to_s3.ex index cb250c6b..89eaa6a9 100644 --- a/lib/mix/tasks/upload_to_s3.ex +++ b/lib/mix/tasks/upload_to_s3.ex @@ -21,52 +21,44 @@ defmodule Mix.Tasks.UploadToS3 do def run([concurrency | args]) do {concurrency, _} = Integer.parse(concurrency) - if Enum.member?(args, "--dev") do - Application.put_env(:philomena, :advert_file_root, "priv/static/system/images/adverts") - Application.put_env(:philomena, :avatar_file_root, "priv/static/system/images/avatars") - Application.put_env(:philomena, :badge_file_root, "priv/static/system/images") - Application.put_env(:philomena, :image_file_root, "priv/static/system/images") - Application.put_env(:philomena, :tag_file_root, "priv/static/system/images") - end - if Enum.member?(args, "--adverts") do - file_root = Application.fetch_env!(:philomena, :advert_file_root) - new_file_root = System.get_env("NEW_ADVERT_FILE_ROOT", "adverts") + file_root = System.get_env("OLD_ADVERT_FILE_ROOT", "priv/static/system/images/adverts") + new_file_root = Application.fetch_env!(:philomena, :advert_file_root) IO.puts "\nAdverts:" - upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, "image") + upload_typical(where(Advert, [a], not is_nil(a.image)), concurrency, file_root, new_file_root, :image) end if Enum.member?(args, "--avatars") do - file_root = Application.fetch_env!(:philomena, :avatar_file_root) - new_file_root = System.get_env("NEW_AVATAR_FILE_ROOT", "avatars") + file_root = System.get_env("OLD_AVATAR_FILE_ROOT", "priv/static/system/images/avatars") + new_file_root = Application.fetch_env!(:philomena, :avatar_file_root) IO.puts "\nAvatars:" - upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, "avatar") + upload_typical(where(User, [u], not is_nil(u.avatar)), concurrency, file_root, new_file_root, :avatar) end if Enum.member?(args, "--badges") do - file_root = Application.fetch_env!(:philomena, :badge_file_root) - new_file_root = System.get_env("NEW_BADGE_FILE_ROOT", "badges") + file_root = System.get_env("OLD_BADGE_FILE_ROOT", "priv/static/system/images") + new_file_root = Application.fetch_env!(:philomena, :badge_file_root) IO.puts "\nBadges:" - upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, "image") + upload_typical(where(Badge, [b], not is_nil(b.image)), concurrency, file_root, new_file_root, :image) end if Enum.member?(args, "--tags") do - file_root = Application.fetch_env!(:philomena, :tag_file_root) - new_file_root = System.get_env("NEW_TAG_FILE_ROOT", "tags") + file_root = System.get_env("OLD_TAG_FILE_ROOT", "priv/static/system/images") + new_file_root = Application.fetch_env!(:philomena, :tag_file_root) IO.puts "\nTags:" - upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, "image") + upload_typical(where(Tag, [t], not is_nil(t.image)), concurrency, file_root, new_file_root, :image) end if Enum.member?(args, "--images") do - # Temporarily adjust the file root so that the thumbs are picked up - file_root = Application.fetch_env!(:philomena, :image_file_root) <> "thumbs" - Application.put_env(:philomena, :image_file_root, file_root) + file_root = Path.join(System.get_env("OLD_IMAGE_FILE_ROOT", "priv/static/system/images"), "thumbs") + new_file_root = Application.fetch_env!(:philomena, :image_file_root) - new_file_root = System.get_env("NEW_IMAGE_FILE_ROOT", "images") + # Temporarily set file root to empty path so we can get the proper prefix + Application.put_env(:philomena, :image_file_root, "") IO.puts "\nImages:" upload_images(where(Image, [i], not is_nil(i.image)), concurrency, file_root, new_file_root) @@ -75,23 +67,28 @@ defmodule Mix.Tasks.UploadToS3 do defp upload_typical(queryable, batch_size, file_root, new_file_root, field_name) do Batch.record_batches(queryable, [batch_size: batch_size], fn models -> - Task.async_stream(models, &upload_typical_model(&1, file_root, new_file_root, field_name)) + models + |> Task.async_stream(&upload_typical_model(&1, file_root, new_file_root, field_name)) + |> Stream.run() IO.write "\r#{hd(models).id}" end) end defp upload_typical_model(model, file_root, new_file_root, field_name) do - path = Path.join(file_root, Map.fetch!(model, field_name)) + field = Map.fetch!(model, field_name) + path = Path.join(file_root, field) - if File.exists?(path) do - put_file(path, Path.join(new_file_root, field_name)) + if File.regular?(path) do + put_file(path, Path.join(new_file_root, field)) end end defp upload_images(queryable, batch_size, file_root, new_file_root) do Batch.record_batches(queryable, [batch_size: batch_size], fn models -> - Task.async_stream(models, &upload_image_model(&1, file_root, new_file_root)) + models + |> Task.async_stream(&upload_image_model(&1, file_root, new_file_root)) + |> Stream.run() IO.write "\r#{hd(models).id}" end) @@ -105,12 +102,14 @@ defmodule Mix.Tasks.UploadToS3 do path = Path.join([file_root, path_prefix, version]) new_path = Path.join([new_file_root, path_prefix, version]) - put_file(path, new_path) + if File.regular?(path) do + put_file(path, new_path) + end end) end defp put_file(path, uploaded_path) do - mime = Mime.file(path) + {_, mime} = Mime.file(path) contents = File.read!(path) S3.put_object(bucket(), uploaded_path, contents, acl: :public_read, content_type: mime)