From f34f06bca57cadd95bb3319d6e1346fd0ff166ba Mon Sep 17 00:00:00 2001 From: 0x24d Date: Fri, 19 Nov 2021 22:41:43 +0000 Subject: [PATCH] Store continuation so that it can be used on the next matching request. This is done because the continuation created for videos sorted by 'oldest' doesn't work after the first 30 videos. The same 30 videos are returned again. The only way to get the next 30 videos, and onwards, is to use the continuation returned in the initial API call. Storing the returned continuation in the db saves having to request each page from 1 to the currently wanted page each time a page other than the first is wanted. --- config/sql/channel_continuations.sql | 23 ++++++++++++ src/invidious.cr | 1 + src/invidious/channels/channels.cr | 38 ++++++++++++++++++++ src/invidious/channels/videos.cr | 31 ++++++++++++++-- src/invidious/yt_backend/extractors_utils.cr | 10 ++++-- 5 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 config/sql/channel_continuations.sql diff --git a/config/sql/channel_continuations.sql b/config/sql/channel_continuations.sql new file mode 100644 index 00000000..629b2aef --- /dev/null +++ b/config/sql/channel_continuations.sql @@ -0,0 +1,23 @@ +-- Table: public.channel_continuations + +-- DROP TABLE public.channel_continuations; + +CREATE TABLE IF NOT EXISTS public.channel_continuations +( + id text NOT NULL, + page integer, + sort_by text, + continuation text, + CONSTRAINT channel_continuations_id_page_sort_by_key UNIQUE (id, page, sort_by) +); + +GRANT ALL ON TABLE public.channel_continuations TO default_user; + +-- Index: public.channel_continuations_id_idx + +-- DROP INDEX public.channel_continuations_id_idx; + +CREATE INDEX IF NOT EXISTS channel_continuations_id_idx + ON public.channel_continuations + USING btree + (id COLLATE pg_catalog."default"); diff --git a/src/invidious.cr b/src/invidious.cr index 21a12ff2..c4ecc275 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -115,6 +115,7 @@ if CONFIG.check_tables check_enum(PG_DB, "privacy", PlaylistPrivacy) check_table(PG_DB, "channels", InvidiousChannel) + check_table(PG_DB, "channel_continuations", ChannelContinuation) check_table(PG_DB, "channel_videos", ChannelVideo) check_table(PG_DB, "playlists", InvidiousPlaylist) check_table(PG_DB, "playlist_videos", PlaylistVideo) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..7d89c1a8 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -8,6 +8,23 @@ struct InvidiousChannel property subscribed : Time? end +struct ChannelContinuation + include DB::Serializable + + property id : String + property page : Int32 = 0 + property sort_by : String = "newest" + property continuation : String + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map(&.name)}} + } + {% end %} + end +end + struct ChannelVideo include DB::Serializable @@ -199,6 +216,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) page = 1 + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: "newest", + continuation: produce_channel_videos_continuation(ucid, auto_generated: auto_generated, v2: true) + }) + + LOGGER.trace("fetch_channel: #{ucid} : page #{page} : Updating or inserting continuation") + + db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = extract_videos(initial_data, author, ucid) @@ -264,6 +293,15 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = extract_videos(initial_data, author, ucid) + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: "newest", + continuation: fetch_continuation_token(initial_data) || "" + }) + db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + count = videos.size videos = videos.map { |video| ChannelVideo.new({ id: video.id, diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 48453bb7..5949345e 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -58,10 +58,35 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so end def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) + if channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, page, sort_by, as: ChannelContinuation) + continuation = channel_continuation.continuation + else + # Manually create the continuation, and insert it into the table, if one does not already exist. + # This should only the case the first time the first page of each 'sort_by' mode is loaded for each channel, + # as all calls to this function with 'page = 1' will get the continuation for the next page (page 2) from the returned data below. + continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: sort_by, + continuation: continuation + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + end - return YoutubeAPI.browse(continuation) + initial_data = YoutubeAPI.browse(continuation) + # Store the returned continuation in the table so that it can be used the next time this function is called requesting that page. + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page + 1, + sort_by: sort_by, + continuation: fetch_continuation_token(initial_data) || "" + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + + return initial_data end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index add5f488..400bc62e 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -58,10 +58,16 @@ def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) # Fetches the continuation token from initial data if initial_data["onResponseReceivedActions"]? continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else + elsif initial_data["contents"]? tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + else + continuation = initial_data["continuationContents"]["gridContinuation"]["continuations"][0]["nextContinuationData"]["continuation"].as_s end - return fetch_continuation_token(continuation_items.as_a) + if continuation_items.nil? + return continuation + else + return fetch_continuation_token(continuation_items.as_a) + end end