Channels: Use innertube to fetch the community tab (#3988)

This commit is contained in:
Samantaz Fox 2023-07-20 08:15:06 +02:00
commit 930b0a158d
No known key found for this signature in database
GPG key ID: F42821059186176E
2 changed files with 34 additions and 46 deletions

View file

@ -1,49 +1,31 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by" # TODO: Add "sort_by"
def fetch_channel_community(ucid, continuation, locale, format, thin_mode) def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") if cursor.nil?
if response.status_code != 200 # Egljb21tdW5pdHk%3D is the protobuf object to load "community"
response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
end
if response.status_code != 200 items = [] of JSON::Any
raise NotFoundException.new("This channel does not exist.") extract_items(initial_data) do |item|
end items << item
ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"]
if !continuation || continuation.empty?
initial_data = extract_initial_data(response.body)
body = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]
if !body
raise InfoException.new("Could not extract community tab.")
end end
else else
continuation = produce_channel_community_continuation(ucid, continuation) continuation = produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
headers = HTTP::Headers.new container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" raise InfoException.new("Can't extract community data") if container.nil?
post_req = {
session_token: session_token,
}
body = YoutubeAPI.browse(continuation) items = container.as_a
body = body.dig?("continuationContents", "itemSectionContinuation") ||
body.dig?("continuationContents", "backstageCommentsContinuation")
if !body
raise InfoException.new("Could not extract continuation.")
end
end end
posts = body["contents"].as_a return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
if message = posts[0]["messageRenderer"]? def extract_channel_community(items, *, ucid, locale, format, thin_mode)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? || error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?) message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || "" .try &.as_s || ""
@ -59,7 +41,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
json.field "authorId", ucid json.field "authorId", ucid
json.field "comments" do json.field "comments" do
json.array do json.array do
posts.each do |post| items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]? post["backstageCommentsContinuation"]?
@ -242,7 +224,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
end end
end end
end end
if cont = posts.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
json.field "continuation", extract_channel_community_cursor(cont.as_s) json.field "continuation", extract_channel_community_cursor(cont.as_s)
end end
end end

View file

@ -608,19 +608,25 @@ private module Extractors
private def self.unpack_section_list(contents) private def self.unpack_section_list(contents)
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
contents.as_a.each do |renderer_container| contents.as_a.each do |item|
renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] if item_section_content = item.dig?("itemSectionRenderer", "contents")
raw_items += self.unpack_item_section(item_section_content)
# Category extraction
if items_container = renderer_container_contents["shelfRenderer"]?
raw_items << renderer_container_contents
next
elsif items_container = renderer_container_contents["gridRenderer"]?
else else
items_container = renderer_container_contents raw_items << item
end end
end
items_container["items"]?.try &.as_a.each do |item| return raw_items
end
private def self.unpack_item_section(contents)
raw_items = [] of JSON::Any
contents.as_a.each do |item|
# Category extraction
if container = item.dig?("gridRenderer", "items") || item.dig?("items")
raw_items += container.as_a
else
raw_items << item raw_items << item
end end
end end