diff --git a/lib/philomena/http.ex b/lib/philomena/http.ex index 12705e89..94570e71 100644 --- a/lib/philomena/http.ex +++ b/lib/philomena/http.ex @@ -1,11 +1,15 @@ defmodule Philomena.Http do + @user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"] + def get!(url, headers \\ [], options \\ []) do + headers = Keyword.merge(@user_agent, headers) options = Keyword.merge(options, proxy: proxy_host()) HTTPoison.get!(url, headers, options) end def head!(url, headers \\ [], options \\ []) do + headers = Keyword.merge(@user_agent, headers) options = Keyword.merge(options, proxy: proxy_host()) HTTPoison.head!(url, headers, options) diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex index f036fbc1..1fab2ba4 100644 --- a/lib/philomena/scrapers/twitter.ex +++ b/lib/philomena/scrapers/twitter.ex @@ -3,7 +3,6 @@ defmodule Philomena.Scrapers.Twitter do @url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?| @script_regex ~r|