From cc51981b050a93138129cf9632d3c50c97cba75a Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Mon, 15 Mar 2021 10:56:15 -0400
Subject: [PATCH] fix twitter scraper

---
 lib/philomena/http.ex             | 2 +-
 lib/philomena/scrapers/twitter.ex | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/philomena/http.ex b/lib/philomena/http.ex
index fa9ac7d8..9767995f 100644
--- a/lib/philomena/http.ex
+++ b/lib/philomena/http.ex
@@ -33,7 +33,7 @@ defmodule Philomena.Http do
         {Tesla.Middleware.Headers,
          [
            {"User-Agent",
-            "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/76.0"}
+            "Mozilla/5.0 (X11; Philomena; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0"}
            | headers
          ]}
       ],
diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex
index 07ecafa7..dde225ef 100644
--- a/lib/philomena/scrapers/twitter.ex
+++ b/lib/philomena/scrapers/twitter.ex
@@ -1,7 +1,7 @@
 defmodule Philomena.Scrapers.Twitter do
-  @gt_regex ~r|document.cookie = decodeURIComponent\("gt=(\d+);|
+  @gt_regex ~r|decodeURIComponent\("gt=(\d+);|
   @url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?|
-  @script_regex ~r|<script type="text/javascript" .*? src="(https://abs.twimg.com/responsive-web/client-web/main\.[\da-z]+\.js)">|
+  @script_regex ~r|="(https://abs.twimg.com/responsive-web/client-web(?:-legacy)?/main\.[\da-z]+\.js)"|
   @bearer_regex ~r|"(AAAAAAAAAAAAA[^"]*)"|
 
   @spec can_handle?(URI.t(), String.t()) :: true | false
@@ -64,7 +64,7 @@ defmodule Philomena.Scrapers.Twitter do
 
   defp extract_guest_token_and_bearer({:ok, %Tesla.Env{body: page}}) do
     [gt] = Regex.run(@gt_regex, page, capture: :all_but_first)
-    [script] = Regex.run(@script_regex, page, capture: :all_but_first)
+    [script | _] = Regex.run(@script_regex, page, capture: :all_but_first)
 
     {:ok, %{body: body}} = Philomena.Http.get(script)