Channels: Use innertube to fetch the community tab (#3988)

This commit is contained in:
Samantaz Fox 2023-07-20 08:15:06 +02:00
commit 930b0a158d
No known key found for this signature in database
GPG Key ID: F42821059186176E
2 changed files with 34 additions and 46 deletions

View File

@ -1,49 +1,31 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
if response.status_code != 200
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
end
def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
if cursor.nil?
# Egljb21tdW5pdHk%3D is the protobuf object to load "community"
initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
if response.status_code != 200
raise NotFoundException.new("This channel does not exist.")
end
ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"]
if !continuation || continuation.empty?
initial_data = extract_initial_data(response.body)
body = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]
if !body
raise InfoException.new("Could not extract community tab.")
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
else
continuation = produce_channel_community_continuation(ucid, continuation)
continuation = produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
headers = HTTP::Headers.new
headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
post_req = {
session_token: session_token,
}
raise InfoException.new("Can't extract community data") if container.nil?
body = YoutubeAPI.browse(continuation)
body = body.dig?("continuationContents", "itemSectionContinuation") ||
body.dig?("continuationContents", "backstageCommentsContinuation")
if !body
raise InfoException.new("Could not extract continuation.")
end
items = container.as_a
end
posts = body["contents"].as_a
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
if message = posts[0]["messageRenderer"]?
def extract_channel_community(items, *, ucid, locale, format, thin_mode)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
@ -59,7 +41,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
json.field "authorId", ucid
json.field "comments" do
json.array do
posts.each do |post|
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
@ -242,7 +224,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
end
end
end
if cont = posts.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
json.field "continuation", extract_channel_community_cursor(cont.as_s)
end
end

View File

@ -608,19 +608,25 @@ private module Extractors
private def self.unpack_section_list(contents)
raw_items = [] of JSON::Any
contents.as_a.each do |renderer_container|
renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0]
# Category extraction
if items_container = renderer_container_contents["shelfRenderer"]?
raw_items << renderer_container_contents
next
elsif items_container = renderer_container_contents["gridRenderer"]?
contents.as_a.each do |item|
if item_section_content = item.dig?("itemSectionRenderer", "contents")
raw_items += self.unpack_item_section(item_section_content)
else
items_container = renderer_container_contents
raw_items << item
end
end
items_container["items"]?.try &.as_a.each do |item|
return raw_items
end
private def self.unpack_item_section(contents)
raw_items = [] of JSON::Any
contents.as_a.each do |item|
# Category extraction
if container = item.dig?("gridRenderer", "items") || item.dig?("items")
raw_items += container.as_a
else
raw_items << item
end
end