PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2024-11-27 13:47:58 +01:00
Code Issues Projects Releases Packages Wiki Activity

Remove extremely outdated redirect follower from DA scraper

Browse source
This commit is contained in:
Liam 2024-06-19 23:57:00 -04:00
parent 29dc68c714
commit 44c160b905
1 changed files with 1 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
lib/philomena_proxy/scrapers/deviantart.ex
View file

@ -31,7 +31,7 @@ defmodule PhilomenaProxy.Scrapers.Deviantart do
@spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result() @spec scrape(URI.t(), Scrapers.url()) :: Scrapers.scrape_result()
def scrape(_uri, url) do def scrape(_uri, url) do
url url
|> follow_redirect(2) |> PhilomenaProxy.Http.get()
|> extract_data!() |> extract_data!()
|> try_intermediary_hires!() |> try_intermediary_hires!()
|> try_new_hires!() |> try_new_hires!()
@ -139,22 +139,4 @@ defmodule PhilomenaProxy.Scrapers.Deviantart do
data data
end end
end end
# Workaround for benoitc/hackney#273
defp follow_redirect(_url, 0), do: nil
defp follow_redirect(url, max_times) do
case PhilomenaProxy.Http.get(url) do
{:ok, %{headers: headers, status: code}} when code in [301, 302] ->
location = Enum.find_value(headers, &location_header/1)
follow_redirect(location, max_times - 1)
response ->
response
end
end
defp location_header({"Location", location}), do: location
defp location_header({"location", location}), do: location
defp location_header(_), do: nil
end end