always use custom user agent

This commit is contained in:
byte[] 2019-11-28 19:34:22 -05:00
parent 85b1f2b1ec
commit 9468ede4e3
3 changed files with 8 additions and 5 deletions

View file

@ -1,11 +1,15 @@
defmodule Philomena.Http do
@user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"]
def get!(url, headers \\ [], options \\ []) do
headers = Keyword.merge(@user_agent, headers)
options = Keyword.merge(options, proxy: proxy_host())
HTTPoison.get!(url, headers, options)
end
def head!(url, headers \\ [], options \\ []) do
headers = Keyword.merge(@user_agent, headers)
options = Keyword.merge(options, proxy: proxy_host())
HTTPoison.head!(url, headers, options)

View file

@ -3,7 +3,6 @@ defmodule Philomena.Scrapers.Twitter do
@url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?|
@script_regex ~r|<script type="text/javascript" .*? src="(https://abs.twimg.com/responsive-web/web/main\.[\da-z]+\.js)">|
@bearer_regex ~r|"(AAAAAAAAAAAAA[^"]*)"|
@user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"]
@spec can_handle?(URI.t(), String.t()) :: true | false
def can_handle?(_uri, url) do
@ -39,7 +38,7 @@ defmodule Philomena.Scrapers.Twitter do
url = "https://twitter.com/#{user}/status/#{status_id}"
{gt, bearer} =
Philomena.Http.get!(mobile_url, @user_agent)
Philomena.Http.get!(mobile_url)
|> Map.get(:body)
|> extract_guest_token_and_bearer()

View file

@ -45,7 +45,7 @@ h1 Reverse Search
= for match <- @images do
tr
td
th
h3 = link "##{match.id}", to: Routes.image_path(@conn, :show, match)
p
= if match.source_url not in [nil, ""] do
@ -54,10 +54,10 @@ h1 Reverse Search
- else
' Unknown source
td
th
= render PhilomenaWeb.ImageView, "_image_container.html", image: match, size: :thumb, conn: @conn
td
th
h3
= match.image_width
| x