Adding Pixiv with referrer mod to ScraperPlug

This commit is contained in:
Chaska 2024-04-24 10:54:31 -05:00
parent 0499bc7605
commit 76b8820aed
5 changed files with 35 additions and 9 deletions

View file

@ -85,6 +85,10 @@
"title": "About",
"url": "/pages/about"
},
{
"title": "Telegram Group",
"url": "https://t.me/+NDKRohflXWs3YzIx"
},
{
"title": "InkBunny",
"url": "https://inkbunny.net/cubfurgay",

View file

@ -15,12 +15,28 @@ defmodule Philomena.Scrapers.Pixiv do
submission = json["body"]
description = submission["illust_details"]["comment"]
images = for x <- submission["illust_details"] do
images = if submission["illust_details"]["manga_a"] do
submission["illust_details"]["manga_a"]
else
[submission["illust_details"]]
end
images = for x <- images do
pre = x["url_small"] || x["url_ss"]
{:ok, %Tesla.Env{status: 200, body: body, headers: headers}}
= Philomena.Http.get(pre, [{"Referer", "https://pixiv.net/"}])
type = headers
|> Enum.into(%{})
|> Map.get("content-type")
%{
url: "#{x["url_big"]}",
camo_url: Camo.Image.image_url(x["url"])
url: x["url"],
camo_url: "data:#{type};base64,#{Base.encode64(body)}"
}
end
%{
source_url: url,
author_name: submission["author_details"]["user_account"],

View file

@ -20,6 +20,15 @@ defmodule PhilomenaWeb.ScraperPlug do
_ ->
conn
headers = if String.contains?(url, "pximg.net") do
[{"Referer", "https://pixiv.net/"}]
else
[]
end
url
|> Philomena.Http.get(headers)
|> maybe_fixup_params(url, opts, conn)
end
end

View file

@ -16,7 +16,7 @@ footer#footer
#serving_info
' Powered by
a> href="https://github.com/philomena-dev/philomena" philomena
a> href="https://github.com/cubfur/philomena" philomena (CubFur Edition)
| (
= render_time(@conn)
| ms)

View file

@ -14,11 +14,8 @@ h1 Staff
.block.block--fixed
p
' Before contacting any of the staff members, you should try to ask your question in our
= link to: "/pages/discord" do
' Discord
' or
= link to: "/pages/irc" do
' IRC channels.
= link to: "https://t.me/+NDKRohflXWs3YzIx" do
' Telegram group.
p Keep in mind that all staff are unpaid volunteers who donate their time and effort into making sure this site remains organized and operational. Please do not harass them, and try to keep your PMs constructive. We will happily answer your questions, however receiving plenty of PMs for no reason gets tiring and impacts our ability to tend to more important matters, so please make sure you actually have a need to contact a staff member before doing so.
.staff-block