From 44c160b905cb7e409b1c0cfc11cd4279cea5d848 Mon Sep 17 00:00:00 2001 From: Liam Date: Wed, 19 Jun 2024 23:57:00 -0400 Subject: [PATCH] Remove extremely outdated redirect follower from DA scraper --- lib/philomena_proxy/scrapers/deviantart.ex | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/lib/philomena_proxy/scrapers/deviantart.ex b/lib/philomena_proxy/scrapers/deviantart.ex index 138d67e1..d292e8aa 100644 --- a/lib/philomena_proxy/scrapers/deviantart.ex +++ b/lib/philomena_proxy/scrapers/deviantart.ex @@ -31,7 +31,7 @@ defmodule PhilomenaProxy.Scrapers.Deviantart do @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result() def scrape(_uri, url) do url - |> follow_redirect(2) + |> PhilomenaProxy.Http.get() |> extract_data!() |> try_intermediary_hires!() |> try_new_hires!() @@ -139,22 +139,4 @@ defmodule PhilomenaProxy.Scrapers.Deviantart do data end end - - # Workaround for benoitc/hackney#273 - defp follow_redirect(_url, 0), do: nil - - defp follow_redirect(url, max_times) do - case PhilomenaProxy.Http.get(url) do - {:ok, %{headers: headers, status: code}} when code in [301, 302] -> - location = Enum.find_value(headers, &location_header/1) - follow_redirect(location, max_times - 1) - - response -> - response - end - end - - defp location_header({"Location", location}), do: location - defp location_header({"location", location}), do: location - defp location_header(_), do: nil end