Add abilty to fetch histortical community posts

src/invidious/channels/community.cr

@@ -1,13 +1,31 @@
 # Fetches channel community posts for the initial page
 def fetch_channel_community(ucid)
   initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
-  return extract_items(initial_data)
+  continuation_token = fetch_continuation_token(initial_data)
+  cursor = continuation_token ? extract_channel_community_cursor(continuation_token) : nil
+
+  return extract_items(initial_data), cursor
 end
 
 # Fetches the next batch of community posts after the given cursor
-def fetch_channel_community(ucid, cursor)
+def fetch_channel_community(ucid, cursor, skip_full_page_check = false)
   continuation = produce_channel_community_continuation(ucid, cursor)
   initial_data = YoutubeAPI.browse(continuation)
+
+  continuation_token = fetch_continuation_token(initial_data)
+  cursor = continuation_token ? extract_channel_community_cursor(continuation_token) : nil
+  items = extract_items(initial_data)
+
+  if skip_full_page_check
+  else
+    # We want at least four items per page
+    until items.size >= 4 || !cursor
+      more_items, cursor = fetch_channel_community(ucid, cursor, skip_full_page_check = true)
+      items = items + more_items
+    end
+  end
+
+  return items, cursor
 end
 
 def produce_channel_community_continuation(ucid, cursor)
@@ -15,6 +33,7 @@ def produce_channel_community_continuation(ucid, cursor)
     "80226972:embedded" => {
       "2:string"  => ucid,
       "3:string"  => cursor || "",
+      "35:string" => "backstage-item-section",
     },
   }
 

src/invidious/helpers/extractors.cr

@@ -517,6 +517,8 @@ private module Extractors
         raw_items = content["items"].as_a
       elsif content = target["continuationItems"]?
         raw_items = content.as_a
+      elsif content = target["itemSectionContinuation"]?
+        raw_items = content["contents"].as_a
       end
 
       return raw_items
@@ -625,6 +627,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
   if unpackaged_data = initial_data["contents"]?.try &.as_h
   elsif unpackaged_data = initial_data["response"]?.try &.as_h
   elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
+  elsif unpackaged_data = initial_data.dig?("onResponseReceivedEndpoints", 0).try &.as_h
   else
     unpackaged_data = initial_data
   end

src/invidious/helpers/helpers.cr

@@ -242,17 +242,22 @@ end
 
 def fetch_continuation_token(items : Array(JSON::Any))
   # Fetches the continuation token from an array of items
-  return items.last["continuationItemRenderer"]?
-    .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
+  return items.last["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
 end
 
 def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
   # Fetches the continuation token from initial data
-  if initial_data["onResponseReceivedActions"]?
-    continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
+  if target = initial_data["onResponseReceivedActions"]? || initial_data["onResponseReceivedEndpoints"]?
+    continuation_items = target.dig(0, "appendContinuationItemsAction", "continuationItems")
   else
     tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
-    continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
+    target = tab.dig("content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents")
+
+    if continuation_items = target[0]["gridRenderer"]?
+      continuation_items = continuation_items["items"]
+    else
+      continuation_items = target
+    end
   end
 
   return fetch_continuation_token(continuation_items.as_a)

src/invidious/routes/channels.cr

@@ -75,14 +75,13 @@ module Invidious::Routes::Channels
     end
 
     if continuation
-      items = fetch_channel_community(ucid, continuation)
+      items, cursor = fetch_channel_community(ucid, continuation)
     else
-      items = fetch_channel_community(ucid)
+      items, cursor = fetch_channel_community(ucid)
     end
 
-    pp items
-
     return env.redirect "/channel/#{channel.ucid}" if !items || items.empty?
+
     templated "community"
   end
 

src/invidious/views/community.ecr

@@ -111,3 +111,13 @@
     </div>
 <%- end%>
 
+<% if cursor %>
+<div class="pure-g h-box v-box">
+    <div class="pure-u-1 pure-u-md-4-5"></div>
+    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
+        <a href="/channel/<%= channel.ucid %>/community?continuation=<%=cursor%>">
+            <%= translate(locale, "Next page") %>
+        </a>
+    </div>
+</div>
+<% end %>