From 9468ede4e3e6f07816bd51d73ef104c4fffee818 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Thu, 28 Nov 2019 19:34:22 -0500 Subject: [PATCH] always use custom user agent --- lib/philomena/http.ex | 4 ++++ lib/philomena/scrapers/twitter.ex | 3 +-- lib/philomena_web/templates/search/reverse/index.html.slime | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/philomena/http.ex b/lib/philomena/http.ex index 12705e89..94570e71 100644 --- a/lib/philomena/http.ex +++ b/lib/philomena/http.ex @@ -1,11 +1,15 @@ defmodule Philomena.Http do + @user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"] + def get!(url, headers \\ [], options \\ []) do + headers = Keyword.merge(@user_agent, headers) options = Keyword.merge(options, proxy: proxy_host()) HTTPoison.get!(url, headers, options) end def head!(url, headers \\ [], options \\ []) do + headers = Keyword.merge(@user_agent, headers) options = Keyword.merge(options, proxy: proxy_host()) HTTPoison.head!(url, headers, options) diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex index f036fbc1..1fab2ba4 100644 --- a/lib/philomena/scrapers/twitter.ex +++ b/lib/philomena/scrapers/twitter.ex @@ -3,7 +3,6 @@ defmodule Philomena.Scrapers.Twitter do @url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?| @script_regex ~r|