diff --git a/assets/js/upload.js b/assets/js/upload.js index 4f1a0d57..f6368568 100644 --- a/assets/js/upload.js +++ b/assets/js/upload.js @@ -20,6 +20,7 @@ function setupImageUpload() { const [ fileField, remoteUrl, scraperError ] = $$('.js-scraper', form); const [ sourceEl, tagsEl, descrEl ] = $$('.js-image-input', form); const fetchButton = $('#js-scraper-preview'); + if (!fetchButton) return; function showImages(images) { clearEl(imgPreviews); diff --git a/lib/philomena/scrapers.ex b/lib/philomena/scrapers.ex index 91c07571..ed88a9bb 100644 --- a/lib/philomena/scrapers.ex +++ b/lib/philomena/scrapers.ex @@ -2,7 +2,7 @@ defmodule Philomena.Scrapers do @scrapers [ Philomena.Scrapers.Deviantart, Philomena.Scrapers.Twitter, - #Philomena.Scrapers.Tumblr, # blocked on benoitc/hackney#566 + Philomena.Scrapers.Tumblr, Philomena.Scrapers.Raw ] @@ -21,4 +21,4 @@ defmodule Philomena.Scrapers do defp unwrap([result]), do: result defp unwrap(_result), do: nil -end \ No newline at end of file +end diff --git a/lib/philomena/scrapers/deviantart.ex b/lib/philomena/scrapers/deviantart.ex index b6c1841e..f86b93df 100644 --- a/lib/philomena/scrapers/deviantart.ex +++ b/lib/philomena/scrapers/deviantart.ex @@ -23,7 +23,7 @@ defmodule Philomena.Scrapers.Deviantart do # artists give you. def scrape(_uri, url) do url - |> Philomena.Http.get!([], follow_redirect: true, max_redirect: 2) + |> follow_redirect(2) |> extract_data!() |> try_intermediary_hires!() |> try_new_hires!() @@ -132,4 +132,21 @@ defmodule Philomena.Scrapers.Deviantart do data end end -end \ No newline at end of file + + # Workaround for benoitc/hackney#273 + defp follow_redirect(_url, 0), do: nil + defp follow_redirect(url, max_times) do + case Philomena.Http.get!(url) do + %HTTPoison.Response{headers: headers, status_code: code} when code in [301, 302] -> + location = Enum.find_value(headers, &location_header/1) + follow_redirect(location, max_times - 1) + + response -> + response + end + end + + defp location_header({"Location", location}), do: location + defp location_header({"location", location}), do: location + defp location_header(_), do: nil +end diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex index b319a8c0..31ffc594 100644 --- a/lib/philomena/scrapers/twitter.ex +++ b/lib/philomena/scrapers/twitter.ex @@ -17,7 +17,7 @@ defmodule Philomena.Scrapers.Twitter do defp extract_data(tweet) do images = tweet["entities"]["media"] - |> Enum.map(&%{url: &1["media_url_https"], camo_url: Camo.Image.image_url(&1["media_url_https"])}) + |> Enum.map(&%{url: &1["media_url_https"] <> ":orig", camo_url: Camo.Image.image_url(&1["media_url_https"])}) %{ source_url: tweet["url"], @@ -62,4 +62,4 @@ defmodule Philomena.Scrapers.Twitter do {gt, bearer} end -end \ No newline at end of file +end diff --git a/lib/philomena_web/plugs/scraper_plug.ex b/lib/philomena_web/plugs/scraper_plug.ex index 24d61544..a094ec9f 100644 --- a/lib/philomena_web/plugs/scraper_plug.ex +++ b/lib/philomena_web/plugs/scraper_plug.ex @@ -2,7 +2,13 @@ defmodule PhilomenaWeb.ScraperPlug do def init(opts), do: opts def call(conn, opts) do + params_name = Keyword.get(opts, :params_name, "image") + params_key = Keyword.get(opts, :params_key, "image") + case conn.params do + %{^params_name => %{^params_key => %Plug.Upload{}}} -> + conn + %{"scraper_cache" => url} -> Philomena.Http.get!(url, [], max_body_length: 30_000_000) |> maybe_fixup_params(opts, conn) @@ -36,4 +42,4 @@ defmodule PhilomenaWeb.ScraperPlug do %{conn | params: updated_params} end defp maybe_fixup_params(_response, _opts, conn), do: conn -end \ No newline at end of file +end