From a35942a1b3c10d53ab0a722620a645d5ffa419b0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 27 Jul 2021 18:56:53 -0700 Subject: [PATCH] Add abilty to fetch histortical community posts --- src/invidious/channels/community.cr | 27 +++++++++++++++++++++++---- src/invidious/helpers/extractors.cr | 3 +++ src/invidious/helpers/helpers.cr | 15 ++++++++++----- src/invidious/routes/channels.cr | 7 +++---- src/invidious/views/community.ecr | 10 ++++++++++ 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 304eefc8..b5babeb8 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -1,20 +1,39 @@ # Fetches channel community posts for the initial page def fetch_channel_community(ucid) initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D") - return extract_items(initial_data) + continuation_token = fetch_continuation_token(initial_data) + cursor = continuation_token ? extract_channel_community_cursor(continuation_token) : nil + + return extract_items(initial_data), cursor end # Fetches the next batch of community posts after the given cursor -def fetch_channel_community(ucid, cursor) +def fetch_channel_community(ucid, cursor, skip_full_page_check = false) continuation = produce_channel_community_continuation(ucid, cursor) initial_data = YoutubeAPI.browse(continuation) + + continuation_token = fetch_continuation_token(initial_data) + cursor = continuation_token ? extract_channel_community_cursor(continuation_token) : nil + items = extract_items(initial_data) + + if skip_full_page_check + else + # We want at least four items per page + until items.size >= 4 || !cursor + more_items, cursor = fetch_channel_community(ucid, cursor, skip_full_page_check = true) + items = items + more_items + end + end + + return items, cursor end def produce_channel_community_continuation(ucid, cursor) object = { "80226972:embedded" => { - "2:string" => ucid, - "3:string" => cursor || "", + "2:string" => ucid, + "3:string" => cursor || "", + "35:string" => "backstage-item-section", }, } diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index 028f4f92..cbf397e3 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -517,6 +517,8 @@ private module Extractors raw_items = content["items"].as_a elsif content = target["continuationItems"]? raw_items = content.as_a + elsif content = target["itemSectionContinuation"]? + raw_items = content["contents"].as_a end return raw_items @@ -625,6 +627,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h + elsif unpackaged_data = initial_data.dig?("onResponseReceivedEndpoints", 0).try &.as_h else unpackaged_data = initial_data end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index d3f17f44..ca5c1705 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -242,17 +242,22 @@ end def fetch_continuation_token(items : Array(JSON::Any)) # Fetches the continuation token from an array of items - return items.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s + return items.last["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s end def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) # Fetches the continuation token from initial data - if initial_data["onResponseReceivedActions"]? - continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] + if target = initial_data["onResponseReceivedActions"]? || initial_data["onResponseReceivedEndpoints"]? + continuation_items = target.dig(0, "appendContinuationItemsAction", "continuationItems") else tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) - continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + target = tab.dig("content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents") + + if continuation_items = target[0]["gridRenderer"]? + continuation_items = continuation_items["items"] + else + continuation_items = target + end end return fetch_continuation_token(continuation_items.as_a) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index eb488e56..4202b82a 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -75,14 +75,13 @@ module Invidious::Routes::Channels end if continuation - items = fetch_channel_community(ucid, continuation) + items, cursor = fetch_channel_community(ucid, continuation) else - items = fetch_channel_community(ucid) + items, cursor = fetch_channel_community(ucid) end - pp items - return env.redirect "/channel/#{channel.ucid}" if !items || items.empty? + templated "community" end diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 1a1b5018..899e6348 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -111,3 +111,13 @@ <%- end%> +<% if cursor %> +
+
+ +
+<% end %> \ No newline at end of file