Store continuation so that it can be used on the next matching request.

This is done because the continuation created for videos sorted by
'oldest' doesn't work after the first 30 videos. The same 30 videos are
returned again.

The only way to get the next 30 videos, and onwards, is to use the
continuation returned in the initial API call. Storing the returned
continuation in the db saves having to request each page from 1 to the
currently wanted page each time a page other than the first is wanted.

config/sql/channel_continuations.sql

@@ -0,0 +1,23 @@
+-- Table: public.channel_continuations
+
+-- DROP TABLE public.channel_continuations;
+
+CREATE TABLE IF NOT EXISTS public.channel_continuations
+(
+  id text NOT NULL,
+  page integer,
+  sort_by text,
+  continuation text,
+  CONSTRAINT channel_continuations_id_page_sort_by_key UNIQUE (id, page, sort_by)
+);
+
+GRANT ALL ON TABLE public.channel_continuations TO default_user;
+
+-- Index: public.channel_continuations_id_idx
+
+-- DROP INDEX public.channel_continuations_id_idx;
+
+CREATE INDEX IF NOT EXISTS channel_continuations_id_idx
+  ON public.channel_continuations
+  USING btree
+  (id COLLATE pg_catalog."default");

src/invidious.cr

@@ -115,6 +115,7 @@ if CONFIG.check_tables
   check_enum(PG_DB, "privacy", PlaylistPrivacy)
 
   check_table(PG_DB, "channels", InvidiousChannel)
+  check_table(PG_DB, "channel_continuations", ChannelContinuation)
   check_table(PG_DB, "channel_videos", ChannelVideo)
   check_table(PG_DB, "playlists", InvidiousPlaylist)
   check_table(PG_DB, "playlist_videos", PlaylistVideo)

src/invidious/channels/channels.cr

@@ -8,6 +8,23 @@ struct InvidiousChannel
   property subscribed : Time?
 end
 
+struct ChannelContinuation
+  include DB::Serializable
+
+  property id : String
+  property page : Int32 = 0
+  property sort_by : String = "newest"
+  property continuation : String
+
+  def to_tuple
+    {% begin %}
+      {
+        {{*@type.instance_vars.map(&.name)}}
+      }
+    {% end %}
+  end
+end
+
 struct ChannelVideo
   include DB::Serializable
 
@@ -199,6 +216,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 
   page = 1
 
+  channel_continuation = ChannelContinuation.new({
+    id: ucid,
+    page: page,
+    sort_by: "newest",
+    continuation: produce_channel_videos_continuation(ucid, auto_generated: auto_generated, v2: true)
+  })
+  
+  LOGGER.trace("fetch_channel: #{ucid} : page #{page} : Updating or inserting continuation")
+
+  db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+   ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
   initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
   videos = extract_videos(initial_data, author, ucid)
@@ -264,6 +293,15 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
       videos = extract_videos(initial_data, author, ucid)
 
+      channel_continuation = ChannelContinuation.new({
+        id: ucid,
+        page: page,
+        sort_by: "newest",
+        continuation: fetch_continuation_token(initial_data) || ""
+      })
+      db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+       ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+
       count = videos.size
       videos = videos.map { |video| ChannelVideo.new({
         id:                 video.id,

src/invidious/channels/videos.cr

@@ -58,10 +58,35 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
 end
 
 def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
-  continuation = produce_channel_videos_continuation(ucid, page,
-    auto_generated: auto_generated, sort_by: sort_by, v2: true)
+  if channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, page, sort_by, as: ChannelContinuation)
+    continuation = channel_continuation.continuation
+  else
+    # Manually create the continuation, and insert it into the table, if one does not already exist.
+    # This should only the case the first time the first page of each 'sort_by' mode is loaded for each channel,
+    # as all calls to this function with 'page = 1' will get the continuation for the next page (page 2) from the returned data below.
+    continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true)
+    channel_continuation = ChannelContinuation.new({
+      id: ucid,
+      page: page,
+      sort_by: sort_by,
+      continuation: continuation
+    })
+    PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+     ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+  end
 
-  return YoutubeAPI.browse(continuation)
+  initial_data = YoutubeAPI.browse(continuation)
+  # Store the returned continuation in the table so that it can be used the next time this function is called requesting that page.
+  channel_continuation = ChannelContinuation.new({
+    id: ucid,
+    page: page + 1,
+    sort_by: sort_by,
+    continuation: fetch_continuation_token(initial_data) || ""
+  })
+  PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \
+   ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple)
+
+  return initial_data
 end
 
 def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")

src/invidious/yt_backend/extractors_utils.cr

@@ -58,10 +58,16 @@ def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
   # Fetches the continuation token from initial data
   if initial_data["onResponseReceivedActions"]?
     continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
-  else
+  elsif initial_data["contents"]?
     tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
     continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
+  else
+    continuation = initial_data["continuationContents"]["gridContinuation"]["continuations"][0]["nextContinuationData"]["continuation"].as_s
   end
 
-  return fetch_continuation_token(continuation_items.as_a)
+  if continuation_items.nil?
+    return continuation
+  else
+    return fetch_continuation_token(continuation_items.as_a)
+  end
 end