diff --git a/config/sql/channel_continuations.sql b/config/sql/channel_continuations.sql new file mode 100644 index 00000000..629b2aef --- /dev/null +++ b/config/sql/channel_continuations.sql @@ -0,0 +1,23 @@ +-- Table: public.channel_continuations + +-- DROP TABLE public.channel_continuations; + +CREATE TABLE IF NOT EXISTS public.channel_continuations +( + id text NOT NULL, + page integer, + sort_by text, + continuation text, + CONSTRAINT channel_continuations_id_page_sort_by_key UNIQUE (id, page, sort_by) +); + +GRANT ALL ON TABLE public.channel_continuations TO default_user; + +-- Index: public.channel_continuations_id_idx + +-- DROP INDEX public.channel_continuations_id_idx; + +CREATE INDEX IF NOT EXISTS channel_continuations_id_idx + ON public.channel_continuations + USING btree + (id COLLATE pg_catalog."default"); diff --git a/src/invidious.cr b/src/invidious.cr index 21a12ff2..c4ecc275 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -115,6 +115,7 @@ if CONFIG.check_tables check_enum(PG_DB, "privacy", PlaylistPrivacy) check_table(PG_DB, "channels", InvidiousChannel) + check_table(PG_DB, "channel_continuations", ChannelContinuation) check_table(PG_DB, "channel_videos", ChannelVideo) check_table(PG_DB, "playlists", InvidiousPlaylist) check_table(PG_DB, "playlist_videos", PlaylistVideo) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..7d89c1a8 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -8,6 +8,23 @@ struct InvidiousChannel property subscribed : Time? end +struct ChannelContinuation + include DB::Serializable + + property id : String + property page : Int32 = 0 + property sort_by : String = "newest" + property continuation : String + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map(&.name)}} + } + {% end %} + end +end + struct ChannelVideo include DB::Serializable @@ -199,6 +216,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) page = 1 + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: "newest", + continuation: produce_channel_videos_continuation(ucid, auto_generated: auto_generated, v2: true) + }) + + LOGGER.trace("fetch_channel: #{ucid} : page #{page} : Updating or inserting continuation") + + db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = extract_videos(initial_data, author, ucid) @@ -264,6 +293,15 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = extract_videos(initial_data, author, ucid) + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: "newest", + continuation: fetch_continuation_token(initial_data) || "" + }) + db.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + count = videos.size videos = videos.map { |video| ChannelVideo.new({ id: video.id, diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 48453bb7..5949345e 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -58,10 +58,35 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so end def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) + if channel_continuation = PG_DB.query_one?("SELECT * FROM channel_continuations WHERE id = $1 AND page = $2 AND sort_by = $3", ucid, page, sort_by, as: ChannelContinuation) + continuation = channel_continuation.continuation + else + # Manually create the continuation, and insert it into the table, if one does not already exist. + # This should only the case the first time the first page of each 'sort_by' mode is loaded for each channel, + # as all calls to this function with 'page = 1' will get the continuation for the next page (page 2) from the returned data below. + continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page, + sort_by: sort_by, + continuation: continuation + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + end - return YoutubeAPI.browse(continuation) + initial_data = YoutubeAPI.browse(continuation) + # Store the returned continuation in the table so that it can be used the next time this function is called requesting that page. + channel_continuation = ChannelContinuation.new({ + id: ucid, + page: page + 1, + sort_by: sort_by, + continuation: fetch_continuation_token(initial_data) || "" + }) + PG_DB.exec("INSERT INTO channel_continuations VALUES ($1, $2, $3, $4) \ + ON CONFLICT (id, page, sort_by) DO UPDATE SET continuation = $4", *channel_continuation.to_tuple) + + return initial_data end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index add5f488..400bc62e 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -58,10 +58,16 @@ def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) # Fetches the continuation token from initial data if initial_data["onResponseReceivedActions"]? continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] - else + elsif initial_data["contents"]? tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] + else + continuation = initial_data["continuationContents"]["gridContinuation"]["continuations"][0]["nextContinuationData"]["continuation"].as_s end - return fetch_continuation_token(continuation_items.as_a) + if continuation_items.nil? + return continuation + else + return fetch_continuation_token(continuation_items.as_a) + end end