PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2025-03-28 06:17:46 +01:00
Code Issues Projects Releases Packages Wiki Activity

Allow scrapers to take extra params and ignore them =D

Browse source
This commit is contained in:
Chaska 2024-04-30 21:11:24 -05:00
parent 2cd8830191
commit 4e60f61e89
3 changed files with 18 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/philomena/scrapers/derpibooru.ex
View file

@ -1,5 +1,5 @@
defmodule Philomena.Scrapers.Derpibooru do
@url_regex ~r|\Ahttps?://derpibooru\.org/images/([0-9]+)|
@url_regex ~r|\A(https\:\/\/derpibooru\.org\/images\/([0-9]+))\?*.+|
@spec can_handle?(URI.t(), String.t()) :: true | false
def can_handle?(_uri, url) do
@ -7,8 +7,11 @@ defmodule Philomena.Scrapers.Derpibooru do
end
def scrape(_uri, url) do
[_, submission_id] = Regex.run(@url_regex, url, capture: :all)
api_url = "https://derpibooru.org/api/v1/json/images/#{submission_id}"
[_, url, submission_id] = Regex.run(@url_regex, url, capture: :all)
api_url =
"https://derpibooru.org/api/v1/json/images/#{submission_id}"
{:ok, %Tesla.Env{status: 200, body: body}} = Philomena.Http.get(api_url)
json = Jason.decode!(body)

8
lib/philomena/scrapers/e621.ex
View file

@ -1,5 +1,5 @@
defmodule Philomena.Scrapers.E621 do
@url_regex ~r|\Ahttps?://e621\.net/posts/([0-9]+)|
@url_regex ~r|\A(https\:\/\/e621\.net\/posts\/([0-9]+))\?*.+|
@spec can_handle?(URI.t(), String.t()) :: true | false
def can_handle?(_uri, url) do
@ -7,7 +7,11 @@ defmodule Philomena.Scrapers.E621 do
end
def scrape(_uri, url) do
api_url = "#{url}.json?login=#{e621_user()}&api_key=#{e621_apikey()}"
[_, url, submission_id] = Regex.run(@url_regex, url, capture: :all)
api_url =
"https://e621.net/posts/#{submission_id}.json?login=#{e621_user()}&api_key=#{e621_apikey()}"
{:ok, %Tesla.Env{status: 200, body: body}} = Philomena.Http.get(api_url)
json = Jason.decode!(body)

9
lib/philomena/scrapers/furbooru.ex
View file

@ -1,5 +1,5 @@
defmodule Philomena.Scrapers.Furbooru do
@url_regex ~r|\Ahttps?://furbooru\.org/images/([0-9]+)|
@url_regex ~r|\A(https\:\/\/furbooru\.org\/images\/([0-9]+))\?*.+|
@spec can_handle?(URI.t(), String.t()) :: true | false
def can_handle?(_uri, url) do
@ -7,8 +7,11 @@ defmodule Philomena.Scrapers.Furbooru do
end
def scrape(_uri, url) do
[_, submission_id] = Regex.run(@url_regex, url, capture: :all)
api_url = "https://furbooru.org/api/v1/json/images/#{submission_id}"
[_, url, submission_id] = Regex.run(@url_regex, url, capture: :all)
api_url =
"https://furbooru.org/api/v1/json/images/#{submission_id}"
{:ok, %Tesla.Env{status: 200, body: body}} = Philomena.Http.get(api_url)
json = Jason.decode!(body)