From b642ae4c5dfe6d9d4102ccc36ce1d0539c653b06 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 11 Aug 2021 19:29:19 -0700 Subject: [PATCH] Add specs for testing item extraction --- spec/extraction_spec.cr | 106 ++++ spec/innertube_extraction_spec.cr | 24 + spec/item_jsons/channel_renderer.cr | 273 +++++++++++ spec/item_jsons/playlist_renderer.cr | 458 ++++++++++++++++++ spec/item_jsons/video_renderer.cr | 284 +++++++++++ .../youtube/renderers/category.cr | 12 +- src/invidious/data_structs/youtube/videos.cr | 2 + src/invidious/helpers/extractors.cr | 3 + src/invidious/helpers/macros.cr | 1 + 9 files changed, 1157 insertions(+), 6 deletions(-) create mode 100644 spec/extraction_spec.cr create mode 100644 spec/innertube_extraction_spec.cr create mode 100644 spec/item_jsons/channel_renderer.cr create mode 100644 spec/item_jsons/playlist_renderer.cr create mode 100644 spec/item_jsons/video_renderer.cr diff --git a/spec/extraction_spec.cr b/spec/extraction_spec.cr new file mode 100644 index 00000000..59525b70 --- /dev/null +++ b/spec/extraction_spec.cr @@ -0,0 +1,106 @@ +require "pg" # Required for DB::Serializable +require "spec" + +# Required for initializing DB::Serializable objects with NamedTuples +require "../src/invidious/helpers/macros" + +# Renderer structs +require "../src/invidious/data_structs/youtube/base" +require "../src/invidious/data_structs/youtube/renderers/*" +require "../src/invidious/data_structs/youtube/videos" # Category obj requires Video struct. + +require "../src/invidious/helpers/extractors.cr" +require "./item_jsons/*" + +describe YouTubeStructs::VideoRenderer do + it "It is able to extract a 'standard' videoRenderer without missing information" do + video = extract_item(JSON.parse(VIDEO_RENDERER_EXAMPLES[0])).as(YouTubeStructs::VideoRenderer) + + video.author.should(eq("Kurzgesagt – In a Nutshell")) + video.description_html.should(eq("")) + video.id.should(eq("E1KkQrFEl2I")) + video.length_seconds.should(eq(665)) + video.live_now.should(eq(false)) + video.paid.should(eq(false)) + video.premiere_timestamp.should(eq(nil)) + video.premium.should(eq(false)) + + # Invidious uses the current time (UTC) to compute a timestamp + # from YouTube's relative upload dates on renderers. + video.published.not_nil!.to_s("%Y-%m-%d").should(eq((Time.utc - 9.months).to_s("%Y-%m-%d"))) + + video.title.should(eq("How Large Can a Bacteria get? Life & Size 3")) + video.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q")) + video.views.should(eq(7324534)) + end +end + +describe YouTubeStructs::ChannelRenderer do + it "It is able to extract a 'standard' channelRenderer without missing information" do + channel = extract_item(JSON.parse(CHANNEL_RENDERER_EXAMPLES[0])).as(YouTubeStructs::ChannelRenderer) + + channel.author.should(eq("Kurzgesagt – In a Nutshell")) + channel.author_thumbnail.should(eq("//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s88-c-k-c0x00ffffff-no-rj-mo")) + channel.auto_generated.should(eq(false)) + channel.description_html.should(eq("Videos explaining things with optimistic nihilism. We are a small team who want to make science look beautiful. Because it is ...")) + channel.subscriber_count.should(eq(15700000)) + channel.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q")) + channel.video_count.should(eq(144)) + end + + it "It is able to extract a channelRenderer without subscription information" do + channel = extract_item(JSON.parse(CHANNEL_RENDERER_EXAMPLES[1])).as(YouTubeStructs::ChannelRenderer) + + channel.author.should(eq("Langfocus")) + channel.author_thumbnail.should(eq("//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s88-c-k-c0x00ffffff-no-rj-mo")) + channel.auto_generated.should(eq(false)) + channel.description_html.should(eq("Sharing my passion for languages and reaching out into the wider world.")) + channel.subscriber_count.should(eq(0)) # Not accurate. This value should ideally be nil in this case + channel.ucid.should(eq("UCNhX3WQEkraW3VHPyup8jkQ")) + channel.video_count.should(eq(165)) + end +end + +describe YouTubeStructs::PlaylistRenderer do + it "It is able to extract a 'standard' playlistRenderer without missing information" do + playlist = extract_item(JSON.parse(PLAYLIST_RENDERER_EXAMPLES[0])).as(YouTubeStructs::PlaylistRenderer) + + playlist.author.should(eq("Kurzgesagt – In a Nutshell")) + playlist.id.should(eq("PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF")) + playlist.thumbnail.should(eq("https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg")) + playlist.title.should(eq("The Universe and Space stuff")) + playlist.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q")) + playlist.video_count.should(eq(32)) + playlist.videos.should(eq([ + {title: "The Largest Black Hole in the Universe - Size Comparison", + id: "0FH9cgRhQ-k", + length_seconds: 824}, + + {title: "How To Terraform Venus (Quickly)", + id: "G-WO-z-QuWI", + length_seconds: 768}, + ])) + end + + it "It is able to extract a playlistRenderer located in a grid, and has no missing information" do + # We'll add the channel name and UCID as a fallback + # as the author information just isn't returned by InnerTube in a gridPlaylistRenderer. + playlist = extract_item( + JSON.parse(PLAYLIST_RENDERER_EXAMPLES[1]), + "Kurzgesagt – In a Nutshell", + "UCsXVk37bltHxD1rDPwtNM8Q" + ).as(YouTubeStructs::PlaylistRenderer) + + playlist.author.should(eq("Kurzgesagt – In a Nutshell")) + playlist.id.should(eq("PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs")) + playlist.thumbnail.should(eq("https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg")) + playlist.title.should(eq("The Existential Crisis Playlist")) + playlist.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q")) + playlist.video_count.should(eq(34)) + playlist.videos.should(eq(Array(YouTubeStructs::PlaylistVideoRenderer).new)) + end +end + +describe YouTubeStructs::Category do + # TODO +end diff --git a/spec/innertube_extraction_spec.cr b/spec/innertube_extraction_spec.cr new file mode 100644 index 00000000..794e1a06 --- /dev/null +++ b/spec/innertube_extraction_spec.cr @@ -0,0 +1,24 @@ +require "pg" +require "kemal" +require "../src/invidious/helpers/logger" +require "../src/invidious/helpers/youtube_api" +require "../src/invidious/helpers/macros" +require "../src/invidious/helpers/extractors" + +# To avoid importing invidious.cr we'll go ahead and define these two constants in here. +LOGGER = Invidious::LogHandler.new(STDOUT, LogLevel::Info) +YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: 100, timeout: 2.0, use_quic: true) + +it "Extracts search results" do + extract_items(YoutubeAPI.search("kurzgesagt", "CABIAA%3D%3D")) +end + +describe "Channel" do + it "Extracts video results" do + extract_items(YoutubeAPI.browse("UCsXVk37bltHxD1rDPwtNM8Q", params: "EgZ2aWRlb3M%3D")).size.should be > 1 + end + + it "Extracts playlist results" do + extract_items(YoutubeAPI.browse("UCsXVk37bltHxD1rDPwtNM8Q", params: "EglwbGF5bGlzdHM%3D")).size.should be > 1 + end +end diff --git a/spec/item_jsons/channel_renderer.cr b/spec/item_jsons/channel_renderer.cr new file mode 100644 index 00000000..952f5723 --- /dev/null +++ b/spec/item_jsons/channel_renderer.cr @@ -0,0 +1,273 @@ +# The following are examples of InnerTube channelRenderers +# +# A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not** +# the channel page itself. +CHANNEL_RENDERER_EXAMPLES = [ + # Standard channel without missing information + {"channelRenderer": { + # Channel ID + "channelId": "UCsXVk37bltHxD1rDPwtNM8Q", + + # Author name. Can only be simpleText.\ + "title": { + "simpleText": "Kurzgesagt – In a Nutshell", + }, + + # Endpoint to arrive on after clicking on renderer + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/user/Kurzgesagt", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl": "/user/Kurzgesagt", + }, + }, + + # Array of thumbnails in increasing quality. + "thumbnail": { + "thumbnails": [ + { + "url": "//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s88-c-k-c0x00ffffff-no-rj-mo", + "width": 88, + "height": 88, + }, + { + "url": "//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s176-c-k-c0x00ffffff-no-rj-mo", + "width": 176, + "height": 176, + }, + ], + }, + + # Description snippet. + "descriptionSnippet": { + "runs": [ + { + "text": "Videos explaining things with optimistic nihilism. We are a small team who want to make science look beautiful. Because it is ...", + }, + ], + }, + + # (short) Author information. + "shortBylineText": { + "runs": [ + { + "text": "Kurzgesagt – In a Nutshell", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/user/Kurzgesagt", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl": "/user/Kurzgesagt", + }, + }, + }, + ], + }, + + # Amount of (public?) videos published on the channel. + "videoCountText": { + "runs": [ + { + "text": "144", + }, + { + "text": " videos", + }, + ], + }, + + # Should the subscribe button be renderers as a Subscribed variant? + # "subscriptionButton": {subscribed": false}, + + # Amount of badges the channel has. IE verified. + "ownerBadges": [ + { + "metadataBadgeRenderer": { + "icon": { + "iconType": "CHECK_CIRCLE_THICK", + }, + "style": "BADGE_STYLE_TYPE_VERIFIED", + "tooltip": "Verified", + "TrackingParams": "", + "accessibilityData": { + "label": "Verified", + }, + }, + }, + ], + + # Amount of subscribers the channel has, in an abbreviated format. + # + # This isn't sent by InnerTube for channels that wishes to hide it. + "subscriberCountText": { + "accessibility": { + "accessibilityData": { + "label": "15.7 million subscribers", + }, + }, + "simpleText": "15.7M subscribers", + }, + + # Subscribe button renderer. Useless for Invidious. + # "subscribeButton": {....}, + + # "TrackingParams": "", + + # (Long) Author information. + "longBylineText": { + "runs": [ + { + "text": "Kurzgesagt – In a Nutshell", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/user/Kurzgesagt", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl": "/user/Kurzgesagt", + }, + }, + }, + ], + }, + }}.to_json, + + # See first channelRenderer for detailed explanation. Besides channel data, the only difference + # between this channelRenderer and the previous one is the lack of an "subscriberCountText" + # as it is hidden on this channel. + {"channelRenderer": { + "channelId": "UCNhX3WQEkraW3VHPyup8jkQ", + "title": { + "simpleText": "Langfocus", + }, + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCNhX3WQEkraW3VHPyup8jkQ", + "canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + }, + }, + "thumbnail": { + "thumbnails": [ + { + "url": "//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s88-c-k-c0x00ffffff-no-rj-mo", + "width": 88, + "height": 88, + }, + { + "url": "//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s176-c-k-c0x00ffffff-no-rj-mo", + "width": 176, + "height": 176, + }, + ], + }, + "descriptionSnippet": { + "runs": [ + { + "text": "Sharing my passion for languages and reaching out into the wider world.", + }, + ], + }, + "shortBylineText": { + "runs": [ + { + "text": "Langfocus", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCNhX3WQEkraW3VHPyup8jkQ", + "canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + }, + }, + }, + ], + }, + "videoCountText": { + "runs": [ + { + "text": "165", + }, + { + "text": " videos", + }, + ], + }, + # "subscriptionButton": {subscribed": false}, + "ownerBadges": [ + { + "metadataBadgeRenderer": { + "icon": { + "iconType": "CHECK_CIRCLE_THICK", + }, + "style": "BADGE_STYLE_TYPE_VERIFIED", + "tooltip": "Verified", + "TrackingParams": "", + "accessibilityData": { + "label": "Verified", + }, + }, + }, + ], + # "subscribeButton": {...}, + # "TrackingParams": "", + "longBylineText": { + "runs": [ + { + "text": "Langfocus", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCNhX3WQEkraW3VHPyup8jkQ", + "canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ", + }, + }, + }, + ], + }, + }}.to_json, +] diff --git a/spec/item_jsons/playlist_renderer.cr b/spec/item_jsons/playlist_renderer.cr new file mode 100644 index 00000000..b4462d25 --- /dev/null +++ b/spec/item_jsons/playlist_renderer.cr @@ -0,0 +1,458 @@ +# The following are examples of InnerTube playlistRenderers +# +# A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself. +PLAYLIST_RENDERER_EXAMPLES = [ + {"playlistRenderer": { + "playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + "title": { + "simpleText": "The Universe and Space stuff", + }, + + # Array of thumbnails in increasing quality, taken from the last few videos within the playlist. + "thumbnails": [ + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg", + "width": 168, + "height": 94, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLBJlY_7z-Jfm-lPgZvzcLsuotYD2g", + "width": 196, + "height": 110, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCPYBEIoBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCollsqaYxSm_va6vSN6oK8mnSFhw", + "width": 246, + "height": 138, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDOCmzlwvvYsaaFO2u8lyWPrZULkw", + "width": 336, + "height": 188, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/G-WO-z-QuWI/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/qEfPBt9dU60/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/gLZJlf5rHVs/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/3mnSDifDSxQ/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + ], + + # Amount of videos in playlist + "videoCount": "32", + + # Endpoint to arrive on after clicking on renderer + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832, + }, + }, + "watchEndpoint": { + "videoId": "0FH9cgRhQ-k", + "playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + "params": "OAI%3D", + # "loggingContext": {...}, + # "watchEndpointSupportedOnesieConfig": {...} + }, + }, + + # Renderer for the view full playlist link. This is stored in a + # runs object inside + # "viewPlaylistText": {...}, + + # (short) Author information + "shortBylineText": { + "runs": [ + { + "text": "Kurzgesagt – In a Nutshell", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/user/Kurzgesagt", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl": "/user/Kurzgesagt", + }, + }, + }, + ], + }, + + # Updated/Published date + "publishedTimeText": { + "simpleText": "Updated 7 days ago", + }, + + # Two or less videos from the playlist. This is used to render preview (text-only) + # next to the playlist on search results. Each content below is a mini videoRenderer + "videos": [ + { + "childVideoRenderer": { + "title": { + "simpleText": "The Largest Black Hole in the Universe - Size Comparison", + }, + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832, + }, + }, + "watchEndpoint": { + "videoId": "0FH9cgRhQ-k", + "playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + # "loggingContext": {...}, + # "watchEndpointSupportedOnesieConfig": {...} + }, + }, + "lengthText": { + "accessibility": { + "accessibilityData": { + "label": "13 minutes, 44 seconds", + }, + }, + "simpleText": "13:44", + }, + "videoId": "0FH9cgRhQ-k", + }, + }, + { + "childVideoRenderer": { + "title": { + "simpleText": "How To Terraform Venus (Quickly)", + }, + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=G-WO-z-QuWI&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832, + }, + }, + "watchEndpoint": { + "videoId": "G-WO-z-QuWI", + "playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF", + # "loggingContext": {...}, + # "watchEndpointSupportedOnesieConfig": {...} + }, + }, + "lengthText": { + "accessibility": { + "accessibilityData": { + "label": "12 minutes, 48 seconds", + }, + }, + "simpleText": "12:48", + }, + "videoId": "G-WO-z-QuWI", + }, + }, + ], + + # Amount of videos in playlist + "videoCountText": { + "runs": [ + { + "text": "32", + }, + { + "text": " videos", + }, + ], + }, + # "TrackingParams": "", + + # Overlay counting amount of videos in playlist + # "thumbnailText": {...}, + + # (Long) Author information + "longBylineText": { + "runs": [ + { + "text": "Kurzgesagt – In a Nutshell", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/user/Kurzgesagt", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse", + }, + }, + "browseEndpoint": { + "browseId": "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl": "/user/Kurzgesagt", + }, + }, + }, + ], + }, + + # Owner badges + "ownerBadges": [ + { + "metadataBadgeRenderer": { + "icon": { + "iconType": "CHECK_CIRCLE_THICK", + }, + "style": "BADGE_STYLE_TYPE_VERIFIED", + "tooltip": "Verified", + "TrackingParams": "", + "accessibilityData": { + "label": "Verified", + }, + }, + }, + ], + + # The actual thumbnail of the playlist + # + # YouTube allows for defining custom ones instead of just using the last video + # in the playlist. As such, that can be accessed from here. + "thumbnailRenderer": { + "playlistVideoThumbnailRenderer": { + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg", + "width": 168, + "height": 94, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLBJlY_7z-Jfm-lPgZvzcLsuotYD2g", + "width": 196, + "height": 110, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCPYBEIoBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCollsqaYxSm_va6vSN6oK8mnSFhw", + "width": 246, + "height": 138, + }, + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDOCmzlwvvYsaaFO2u8lyWPrZULkw", + "width": 336, + "height": 188, + }, + ], + }, + }, + }, + + # Thumbnail overlays such as the play all button or the video count. + # "thumbnailOverlays": [] + }}.to_json, + + # Playlists rendered on a grid has a slightly different format + # + # IE lack of author information + {"gridPlaylistRenderer": { + "playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs", + + # Playlist thumbnail in ascending quality + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg", + "width": 480, + "height": 270, + }, + ], + }, + + # Playlist title and endpoint it redirects to on click + "title": { + "runs": [ + { + "text": "The Existential Crisis Playlist", + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832, + }, + }, + "watchEndpoint": { + "videoId": "0FH9cgRhQ-k", + "playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs", + "params": "OAI%3D", + # "loggingContext": {...}, + # "watchEndpointSupportedOnesieConfig": {...} + }, + }, + }, + ], + }, + + # Video count text in format + "videoCountText": { + "runs": [ + { + "text": "34", + }, + { + "text": " videos", + }, + ], + }, + + # Endpoint to arrive on after clicking on renderer + "navigationEndpoint": { + "clickTrackingParams": "", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832, + }, + }, + "watchEndpoint": { + "videoId": "0FH9cgRhQ-k", + "playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs", + "params": "OAI%3D", + # "loggingContext": {...}, + # "watchEndpointSupportedOnesieConfig": {...} + }, + }, + + # Shortened video count text. + "videoCountShortText": { + "simpleText": "34", + }, + # "TrackingParams": "", + + # Array of thumbnails in increasing quality, taken from the last few videos within the playlist. + "sidebarThumbnails": [ + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/JXeJANDKwDc/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/Jzfpyo-q-RM/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/qEfPBt9dU60/default.jpg", + "width": 43, + "height": 20, + }, + ], + }, + ], + + # Renderer for playlist size overlay on thumbnail + "thumbnailText": { + "runs": [ + { + "text": "34", + "bold": true, + }, + { + "text": " videos", + }, + ], + }, + + # Amount of badges the channel has. IE verified. + "ownerBadges": [ + { + "metadataBadgeRenderer": { + "icon": { + "iconType": "CHECK_CIRCLE_THICK", + }, + "style": "BADGE_STYLE_TYPE_VERIFIED", + "tooltip": "Verified", + "TrackingParams": "", + "accessibilityData": { + "label": "Verified", + }, + }, + }, + ], + + # Playlist thumbnail in ascending quality + # + # TODO find difference between this and "thumbnail" object. + "thumbnailRenderer": { + "playlistVideoThumbnailRenderer": { + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg", + "width": 480, + "height": 270, + }, + ], + }, + }, + }, + + # Thumbnail overlays such as the play all button or the video count. + # "thumbnailOverlays": [...], + + # Renderer for the view full playlist link. This is stored in a + # runs object inside + # "viewPlaylistText": {...} + }}.to_json, +] diff --git a/spec/item_jsons/video_renderer.cr b/spec/item_jsons/video_renderer.cr new file mode 100644 index 00000000..d5b6a06b --- /dev/null +++ b/spec/item_jsons/video_renderer.cr @@ -0,0 +1,284 @@ +# The following are examples of InnerTube videoRenderers +# +# A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** +# the watchable video itself. +VIDEO_RENDERER_EXAMPLES = [ + {"videoRenderer" => { + # Video ID + "videoId" => "E1KkQrFEl2I", + # Array of thumbnails in increasing quality. + "thumbnail" => { + "thumbnails" => [ + { + "url" => "https://i.ytimg.com/vi/E1KkQrFEl2I/hq720.jpg?sqp=-oaymwEjCOgCEMoBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLAE7cGsAxbjoQIKa04sXkfF9nTlzw", + "width" => 360, + "height" => 202, + }, + { + "url" => "https://i.ytimg.com/vi/E1KkQrFEl2I/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBnetdf_Lj9C6XpUuIVDV0mn7B2ew", + "width" => 720, + "height" => 404, + }, + ], + }, + # Title. Can also be simpleText + "title" => { + "runs" => [ + { + "text" => "How Large Can a Bacteria get? Life & Size 3", + }, + ], + "accessibility" => { + "accessibilityData" => { + "label" => "How Large Can a Bacteria get? Life & Size 3 by Kurzgesagt – In a Nutshell 9 months ago 11 minutes, 5 seconds 7,324,534 views", + }, + }, + }, + + # (Long) Author information. + "longBylineText" => { + "runs" => [ + { + "text" => "Kurzgesagt – In a Nutshell", + "navigationEndpoint" => { + "clickTrackingParams" => "", + "commandMetadata" => { + "webCommandMetadata" => { + "url" => "/user/Kurzgesagt", + "webPageType" => "WEB_PAGE_TYPE_CHANNEL", + "rootVe" => 3611, + "apiUrl" => "/youtubei/v1/browse", + }, + }, + "browseEndpoint" => { + "browseId" => "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl" => "/user/Kurzgesagt", + }, + }, + }, + ], + }, + + # Published date + # + # For live videos (and possibly recently premiered videos) there is no published information. + # Instead, in its place is the amount of people currently watching. + "publishedTimeText" => { + "simpleText" => "9 months ago", + }, + + # Video Length (locale specific?) + "lengthText" => { + "accessibility" => { + "accessibilityData" => { + "label" => "11 minutes, 5 seconds", + }, + }, + "simpleText" => "11:05", + }, + + # View count (locale specific?) + # + # Typically views are stored under a "simpleText" in the "viewCountText". However, for + # livestreams and premiered it is stored under a "runs" array: [{"text" =>123}, {"text" => "watching"}] + # + # When view count is disabled the "viewCountText" is not present on InnerTube data. + "viewCountText" => { + "simpleText" => "7,324,534 views", + }, + + # Endpoint to arrive on after clicking on renderer + "navigationEndpoint" => { + "clickTrackingParams" => "", + "commandMetadata" => { + "webCommandMetadata" => { + "url" => "/watch?v=E1KkQrFEl2I", + "webPageType" => "WEB_PAGE_TYPE_WATCH", + "rootVe" => 3832, + }, + }, + "watchEndpoint" => { + "videoId" => "E1KkQrFEl2I", + "params" => "qgMKa3Vyemdlc2FndLoDCgj0juTts8vxj1S6AwoI4Lz4wObkpqh0ugMLCLOGu73S_MCcuwG6AwoI6Nmyi6a2-8ZgugMKCImerNeaqIrWdroDHhIcUkRDTVVDc1hWazM3Ymx0SHhEMXJEUHd0Tk04UboDCgji8sL8s9_j8hu6AwoIroO-kuTyidxhugMLCJSWjb7iwfS03gG6AwsIrNvJxeruqv_0AboDCwiRr7iB0-b93uEBugMKCNTbv-uz04eIEroDCwii0dvbjej74ssBugMLCNbouPHd55iEjAG6AwoIpvvOyc3pwtVCugMLCJbux5v_3NKIrgG6AwsIrJPl67y13v6QAboDCwipjcSCw9Xl6qgB8gMFDSaq1Tw%3D", + "watchEndpointSupportedOnesieConfig" => { + "html5PlaybackOnesieConfig" => { + "commonConfig" => { + "url" => "https://r1---sn-nx57ynlk.googlevideo.com/initplayback?source=youtube&orc=1&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&msp=1&odeak=1&odepv=1&osfc=1&ip=198.54.131.169&id=1352a442b1449762&initcwndbps=2022500&mt=1628653969&oweuc=&pxtags=Cg4KAnR4EggyNDAyNzcwNg&rxtags=Cg4KAnR4EggyNDAyNzcwMw%2CCg4KAnR4EggyNDAyNzcwNA%2CCg4KAnR4EggyNDAyNzcwNQ%2CCg4KAnR4EggyNDAyNzcwNg", + }, + }, + }, + }, + }, + + # Video badges. IE Live, CC, etc + "badges" => [ + { + "metadataBadgeRenderer" => { + "style" => "BADGE_STYLE_TYPE_SIMPLE", + "label" => "CC", + "trackingParams" => "", + "accessibilityData" => { + "label" => "Closed captions", + }, + }, + }, + ], + + # Author badges + "ownerBadges" => [ + { + "metadataBadgeRenderer" => { + "icon" => { + "iconType" => "CHECK_CIRCLE_THICK", + }, + "style" => "BADGE_STYLE_TYPE_VERIFIED", + "tooltip" => "Verified", + "trackingParams" => "", + "accessibilityData" => { + "label" => "Verified", + }, + }, + }, + ], + + # Author name + "ownerText" => { + "runs" => [ + { + "text" => "Kurzgesagt – In a Nutshell", + "navigationEndpoint" => { + "clickTrackingParams" => "", + "commandMetadata" => { + "webCommandMetadata" => { + "url" => "/user/Kurzgesagt", + "webPageType" => "WEB_PAGE_TYPE_CHANNEL", + "rootVe" => 3611, + "apiUrl" => "/youtubei/v1/browse", + }, + }, + "browseEndpoint" => { + "browseId" => "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl" => "/user/Kurzgesagt", + }, + }, + }, + ], + }, + + # (Long) Author information. + # TODO find difference between short and long BylineText + "shortBylineText" => { + "runs" => [ + { + "text" => "Kurzgesagt – In a Nutshell", + "navigationEndpoint" => { + "clickTrackingParams" => "", + "commandMetadata" => { + "webCommandMetadata" => { + "url" => "/user/Kurzgesagt", + "webPageType" => "WEB_PAGE_TYPE_CHANNEL", + "rootVe" => 3611, + "apiUrl" => "/youtubei/v1/browse", + }, + }, + "browseEndpoint" => { + "browseId" => "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl" => "/user/Kurzgesagt", + }, + }, + }, + ], + }, + # "trackingParams" => "", + # "showActionMenu" => false, + "shortViewCountText" => { + "accessibility" => { + "accessibilityData" => { + "label" => "7.3 million views", + }, + }, + "simpleText" => "7.3M views", + }, + # "menu" : {...} | renderer for 3 dot menu + + # Channel pfp renderer. Also unused on Invidious + "channelThumbnailSupportedRenderers" => { + "channelThumbnailWithLinkRenderer" => { + "thumbnail" => { + "thumbnails" => [ + { + "url" => "https://yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s68-c-k-c0x00ffffff-no-rj", + "width" => 68, + "height" => 68, + }, + ], + }, + "navigationEndpoint" => { + "clickTrackingParams" => "", + "commandMetadata" => { + "webCommandMetadata" => { + "url" => "/user/Kurzgesagt", + "webPageType" => "WEB_PAGE_TYPE_CHANNEL", + "rootVe" => 3611, + "apiUrl" => "/youtubei/v1/browse", + }, + }, + "browseEndpoint" => { + "browseId" => "UCsXVk37bltHxD1rDPwtNM8Q", + "canonicalBaseUrl" => "/user/Kurzgesagt", + }, + }, + "accessibility" => { + "accessibilityData" => { + "label" => "Go to channel", + }, + }, + }, + }, + + # Provides the overlays on the thumbnails. This is currently + # used as an fallback for the "lengthText" attribute when that + # doesn't exist. + "thumbnailOverlays" => [ + { + "thumbnailOverlayTimeStatusRenderer" => { + "text" => { + "accessibility" => { + "accessibilityData" => { + "label" => "11 minutes, 5 seconds", + }, + }, + "simpleText" => "11:05", + }, + "style" => "DEFAULT", + }, + }, + # Renderer for watch later, add to playlist, etc overlay buttons on YouTube. + # Each separate btn has a different renderer + # {"thumbnailOverlayToggleButtonRenderer" => {...}} + + # thumbnailOverlayNowPlayingRenderer: {...} | Renders "Now playing" + ], + + # Description snippet + "detailedMetadataSnippets" => [ + { + "snippetText" => { + "runs" => [ + { + "text" => "In and out, in and out. Staying alive is about doing things. This very second, your cells are combusting glucose molecules with ...", + }, + ], + }, + "snippetHoverText" => { + "runs" => [ + { + "text" => "From the video description", + }, + ], + }, + "maxOneLine" => false, + }, + ], + }}.to_json, +] diff --git a/src/invidious/data_structs/youtube/renderers/category.cr b/src/invidious/data_structs/youtube/renderers/category.cr index 92f512c8..3878903b 100644 --- a/src/invidious/data_structs/youtube/renderers/category.cr +++ b/src/invidious/data_structs/youtube/renderers/category.cr @@ -18,14 +18,14 @@ module YouTubeStructs property description_html : String property badges : Array(Tuple(String, String))? - # Extracts all renderers out of the category's contents. - def extract_renderers() - target = [] of Renderer + # Extracts all renderers out of the category's contents. + def extract_renderers + target = [] of Renderer - @contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } + @contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } - return target - end + return target + end def to_json(locale, json : JSON::Builder) json.object do diff --git a/src/invidious/data_structs/youtube/videos.cr b/src/invidious/data_structs/youtube/videos.cr index a8e2e8bd..35bcc8a0 100644 --- a/src/invidious/data_structs/youtube/videos.cr +++ b/src/invidious/data_structs/youtube/videos.cr @@ -1,3 +1,5 @@ +require "./caption" + module YouTubeStructs # Converter to serialize first level JSON data as methods for the videos struct module VideoJSONConverter diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index 05bea517..798e9c10 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -66,6 +66,9 @@ private module Parsers # TODO change default value to nil and typical encoding type to tuple storing type (watchers, views, etc) # and count view_count = item_contents.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + + # TODO YouTube seems to have removed the description_html snippet and replaced it with "snippetText" + # inside the detailedMetadataSnippets attribute description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || "" # The length information *should* only always exist in "lengthText". However, the legacy Invidious code diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index 75df1612..01dcfb6b 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -9,6 +9,7 @@ module DB::Serializable }} end + # Initialize DB::Serializable descendants via NamedTuples def initialize(tuple) \{% for var in @type.instance_vars %} \{% ann = var.annotation(::DB::Field) %}