PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2024-11-27 21:47:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

always use custom user agent

Browse source
This commit is contained in:
byte[] 2019-11-28 19:34:22 -05:00
parent 85b1f2b1ec
commit 9468ede4e3
3 changed files with 8 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/philomena/http.ex
View file

@ -1,11 +1,15 @@
defmodule Philomena.Http do defmodule Philomena.Http do
@user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"]
def get!(url, headers \\ [], options \\ []) do def get!(url, headers \\ [], options \\ []) do
headers = Keyword.merge(@user_agent, headers)
options = Keyword.merge(options, proxy: proxy_host()) options = Keyword.merge(options, proxy: proxy_host())
HTTPoison.get!(url, headers, options) HTTPoison.get!(url, headers, options)
end end
def head!(url, headers \\ [], options \\ []) do def head!(url, headers \\ [], options \\ []) do
headers = Keyword.merge(@user_agent, headers)
options = Keyword.merge(options, proxy: proxy_host()) options = Keyword.merge(options, proxy: proxy_host())
HTTPoison.head!(url, headers, options) HTTPoison.head!(url, headers, options)

3
lib/philomena/scrapers/twitter.ex
View file

@ -3,7 +3,6 @@ defmodule Philomena.Scrapers.Twitter do
@url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?| @url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?|
@script_regex ~r|<script type="text/javascript" .*? src="(https://abs.twimg.com/responsive-web/web/main\.[\da-z]+\.js)">| @script_regex ~r|<script type="text/javascript" .*? src="(https://abs.twimg.com/responsive-web/web/main\.[\da-z]+\.js)">|
@bearer_regex ~r|"(AAAAAAAAAAAAA[^"]*)"| @bearer_regex ~r|"(AAAAAAAAAAAAA[^"]*)"|
@user_agent ["User-Agent": "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"]
@spec can_handle?(URI.t(), String.t()) :: true | false @spec can_handle?(URI.t(), String.t()) :: true | false
def can_handle?(_uri, url) do def can_handle?(_uri, url) do
@ -39,7 +38,7 @@ defmodule Philomena.Scrapers.Twitter do
url = "https://twitter.com/#{user}/status/#{status_id}" url = "https://twitter.com/#{user}/status/#{status_id}"
{gt, bearer} = {gt, bearer} =
Philomena.Http.get!(mobile_url, @user_agent) Philomena.Http.get!(mobile_url)
|> Map.get(:body) |> Map.get(:body)
|> extract_guest_token_and_bearer() |> extract_guest_token_and_bearer()

6
lib/philomena_web/templates/search/reverse/index.html.slime
View file

@ -45,7 +45,7 @@ h1 Reverse Search
= for match <- @images do = for match <- @images do
tr tr
td th
h3 = link "##{match.id}", to: Routes.image_path(@conn, :show, match) h3 = link "##{match.id}", to: Routes.image_path(@conn, :show, match)
p p
= if match.source_url not in [nil, ""] do = if match.source_url not in [nil, ""] do
@ -54,10 +54,10 @@ h1 Reverse Search
- else - else
' Unknown source ' Unknown source
td th
= render PhilomenaWeb.ImageView, "_image_container.html", image: match, size: :thumb, conn: @conn = render PhilomenaWeb.ImageView, "_image_container.html", image: match, size: :thumb, conn: @conn
td th
h3 h3
= match.image_width = match.image_width
| x | x