PonyMirrors/philomena
1
0
Fork 0
mirror of https://github.com/philomena-dev/philomena.git synced 2024-11-23 12:08:00 +01:00
Code Issues Projects Releases Packages Wiki Activity

Fix errors in bsky scraper

Browse source
This commit is contained in:
Liam 2024-11-15 08:55:33 -05:00
parent febcdfa62a
commit 282b8b7322
1 changed files with 36 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
lib/philomena_proxy/scrapers/bluesky.ex
View file

@ -19,28 +19,25 @@ defmodule PhilomenaProxy.Scrapers.Bluesky do
def scrape(_uri, url) do def scrape(_uri, url) do
[handle, id] = Regex.run(@url_regex, url, capture: :all_but_first) [handle, id] = Regex.run(@url_regex, url, capture: :all_but_first)
did = did = fetch_did(handle)
if String.starts_with?(handle, "did:") do
handle
else
api_url_resolve_handle =
"https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}"
PhilomenaProxy.Http.get(api_url_resolve_handle) |> json!() |> Map.fetch!(:did)
end
api_url_get_posts = api_url_get_posts =
"https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}" "https://public.api.bsky.app/xrpc/app.bsky.feed.getPosts?uris=at://#{did}/app.bsky.feed.post/#{id}"
post_json = PhilomenaProxy.Http.get(api_url_get_posts) |> json!() |> Map.fetch!(:posts) |> hd post_json =
api_url_get_posts
|> PhilomenaProxy.Http.get()
|> json!()
|> Map.fetch!("posts")
|> hd()
%{ %{
source_url: url, source_url: url,
author_name: post_json["author"]["handle"], author_name: domain_first_component(post_json["author"]["handle"]),
description: post_json["record"]["text"], description: post_json["record"]["text"],
images: images:
post_json["embed"]["images"] Enum.map(
|> Enum.map( post_json["embed"]["images"],
&%{ &%{
url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern), url: String.replace(&1["fullsize"], @fullsize_image_regex, @blob_image_url_pattern),
camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"]) camo_url: PhilomenaProxy.Camo.image_url(&1["thumb"])
@ -49,5 +46,31 @@ defmodule PhilomenaProxy.Scrapers.Bluesky do
} }
end end
defp fetch_did(handle) do
case handle do
<<"did:", _rest::binary>> ->
handle
_ ->
api_url_resolve_handle =
"https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=#{handle}"
api_url_resolve_handle
|> PhilomenaProxy.Http.get()
|> json!()
|> Map.fetch!("did")
end
end
defp domain_first_component(domain) do
case String.split(domain, ".") do
[name | _] ->
name
_ ->
domain
end
end
defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body) defp json!({:ok, %{body: body, status: 200}}), do: Jason.decode!(body)
end end