iv-org-mirror/invidious-copy-2022-04-11
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2022-04-11.git synced 2024-08-15 00:43:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add specs for testing item extraction

Browse source
This commit is contained in:
syeopite 2021-08-11 19:29:19 -07:00
parent df1e4888cd
commit b642ae4c5d
No known key found for this signature in database
GPG key ID: 6FA616E5A5294A82
9 changed files with 1157 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

106
spec/extraction_spec.cr Normal file
View file

@ -0,0 +1,106 @@
require "pg" # Required for DB::Serializable
require "spec"
# Required for initializing DB::Serializable objects with NamedTuples
require "../src/invidious/helpers/macros"
# Renderer structs
require "../src/invidious/data_structs/youtube/base"
require "../src/invidious/data_structs/youtube/renderers/*"
require "../src/invidious/data_structs/youtube/videos" # Category obj requires Video struct.
require "../src/invidious/helpers/extractors.cr"
require "./item_jsons/*"
describe YouTubeStructs::VideoRenderer do
it "It is able to extract a 'standard' videoRenderer without missing information" do
video = extract_item(JSON.parse(VIDEO_RENDERER_EXAMPLES[0])).as(YouTubeStructs::VideoRenderer)
video.author.should(eq("Kurzgesagt – In a Nutshell"))
video.description_html.should(eq(""))
video.id.should(eq("E1KkQrFEl2I"))
video.length_seconds.should(eq(665))
video.live_now.should(eq(false))
video.paid.should(eq(false))
video.premiere_timestamp.should(eq(nil))
video.premium.should(eq(false))
# Invidious uses the current time (UTC) to compute a timestamp
# from YouTube's relative upload dates on renderers.
video.published.not_nil!.to_s("%Y-%m-%d").should(eq((Time.utc - 9.months).to_s("%Y-%m-%d")))
video.title.should(eq("How Large Can a Bacteria get? Life & Size 3"))
video.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q"))
video.views.should(eq(7324534))
end
end
describe YouTubeStructs::ChannelRenderer do
it "It is able to extract a 'standard' channelRenderer without missing information" do
channel = extract_item(JSON.parse(CHANNEL_RENDERER_EXAMPLES[0])).as(YouTubeStructs::ChannelRenderer)
channel.author.should(eq("Kurzgesagt – In a Nutshell"))
channel.author_thumbnail.should(eq("//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s88-c-k-c0x00ffffff-no-rj-mo"))
channel.auto_generated.should(eq(false))
channel.description_html.should(eq("Videos explaining things with optimistic nihilism. We are a small team who want to make science look beautiful. Because it is ..."))
channel.subscriber_count.should(eq(15700000))
channel.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q"))
channel.video_count.should(eq(144))
end
it "It is able to extract a channelRenderer without subscription information" do
channel = extract_item(JSON.parse(CHANNEL_RENDERER_EXAMPLES[1])).as(YouTubeStructs::ChannelRenderer)
channel.author.should(eq("Langfocus"))
channel.author_thumbnail.should(eq("//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s88-c-k-c0x00ffffff-no-rj-mo"))
channel.auto_generated.should(eq(false))
channel.description_html.should(eq("Sharing my passion for languages and reaching out into the wider world."))
channel.subscriber_count.should(eq(0)) # Not accurate. This value should ideally be nil in this case
channel.ucid.should(eq("UCNhX3WQEkraW3VHPyup8jkQ"))
channel.video_count.should(eq(165))
end
end
describe YouTubeStructs::PlaylistRenderer do
it "It is able to extract a 'standard' playlistRenderer without missing information" do
playlist = extract_item(JSON.parse(PLAYLIST_RENDERER_EXAMPLES[0])).as(YouTubeStructs::PlaylistRenderer)
playlist.author.should(eq("Kurzgesagt – In a Nutshell"))
playlist.id.should(eq("PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF"))
playlist.thumbnail.should(eq("https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg"))
playlist.title.should(eq("The Universe and Space stuff"))
playlist.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q"))
playlist.video_count.should(eq(32))
playlist.videos.should(eq([
{title: "The Largest Black Hole in the Universe - Size Comparison",
id: "0FH9cgRhQ-k",
length_seconds: 824},
{title: "How To Terraform Venus (Quickly)",
id: "G-WO-z-QuWI",
length_seconds: 768},
]))
end
it "It is able to extract a playlistRenderer located in a grid, and has no missing information" do
# We'll add the channel name and UCID as a fallback
# as the author information just isn't returned by InnerTube in a gridPlaylistRenderer.
playlist = extract_item(
JSON.parse(PLAYLIST_RENDERER_EXAMPLES[1]),
"Kurzgesagt – In a Nutshell",
"UCsXVk37bltHxD1rDPwtNM8Q"
).as(YouTubeStructs::PlaylistRenderer)
playlist.author.should(eq("Kurzgesagt – In a Nutshell"))
playlist.id.should(eq("PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs"))
playlist.thumbnail.should(eq("https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg"))
playlist.title.should(eq("The Existential Crisis Playlist"))
playlist.ucid.should(eq("UCsXVk37bltHxD1rDPwtNM8Q"))
playlist.video_count.should(eq(34))
playlist.videos.should(eq(Array(YouTubeStructs::PlaylistVideoRenderer).new))
end
end
describe YouTubeStructs::Category do
# TODO
end

24
spec/innertube_extraction_spec.cr Normal file
View file

@ -0,0 +1,24 @@
require "pg"
require "kemal"
require "../src/invidious/helpers/logger"
require "../src/invidious/helpers/youtube_api"
require "../src/invidious/helpers/macros"
require "../src/invidious/helpers/extractors"
# To avoid importing invidious.cr we'll go ahead and define these two constants in here.
LOGGER = Invidious::LogHandler.new(STDOUT, LogLevel::Info)
YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: 100, timeout: 2.0, use_quic: true)
it "Extracts search results" do
extract_items(YoutubeAPI.search("kurzgesagt", "CABIAA%3D%3D"))
end
describe "Channel" do
it "Extracts video results" do
extract_items(YoutubeAPI.browse("UCsXVk37bltHxD1rDPwtNM8Q", params: "EgZ2aWRlb3M%3D")).size.should be > 1
end
it "Extracts playlist results" do
extract_items(YoutubeAPI.browse("UCsXVk37bltHxD1rDPwtNM8Q", params: "EglwbGF5bGlzdHM%3D")).size.should be > 1
end
end

273
spec/item_jsons/channel_renderer.cr Normal file
View file

@ -0,0 +1,273 @@
# The following are examples of InnerTube channelRenderers
#
# A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not**
# the channel page itself.
CHANNEL_RENDERER_EXAMPLES = [
# Standard channel without missing information
{"channelRenderer": {
# Channel ID
"channelId": "UCsXVk37bltHxD1rDPwtNM8Q",
# Author name. Can only be simpleText.\
"title": {
"simpleText": "Kurzgesagt – In a Nutshell",
},
# Endpoint to arrive on after clicking on renderer
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/user/Kurzgesagt",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl": "/user/Kurzgesagt",
},
},
# Array of thumbnails in increasing quality.
"thumbnail": {
"thumbnails": [
{
"url": "//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s88-c-k-c0x00ffffff-no-rj-mo",
"width": 88,
"height": 88,
},
{
"url": "//yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s176-c-k-c0x00ffffff-no-rj-mo",
"width": 176,
"height": 176,
},
],
},
# Description snippet.
"descriptionSnippet": {
"runs": [
{
"text": "Videos explaining things with optimistic nihilism. We are a small team who want to make science look beautiful. Because it is ...",
},
],
},
# (short) Author information.
"shortBylineText": {
"runs": [
{
"text": "Kurzgesagt – In a Nutshell",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/user/Kurzgesagt",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl": "/user/Kurzgesagt",
},
},
},
],
},
# Amount of (public?) videos published on the channel.
"videoCountText": {
"runs": [
{
"text": "144",
},
{
"text": " videos",
},
],
},
# Should the subscribe button be renderers as a Subscribed variant?
# "subscriptionButton": {subscribed": false},
# Amount of badges the channel has. IE verified.
"ownerBadges": [
{
"metadataBadgeRenderer": {
"icon": {
"iconType": "CHECK_CIRCLE_THICK",
},
"style": "BADGE_STYLE_TYPE_VERIFIED",
"tooltip": "Verified",
"TrackingParams": "",
"accessibilityData": {
"label": "Verified",
},
},
},
],
# Amount of subscribers the channel has, in an abbreviated format.
#
# This isn't sent by InnerTube for channels that wishes to hide it.
"subscriberCountText": {
"accessibility": {
"accessibilityData": {
"label": "15.7 million subscribers",
},
},
"simpleText": "15.7M subscribers",
},
# Subscribe button renderer. Useless for Invidious.
# "subscribeButton": {....},
# "TrackingParams": "",
# (Long) Author information.
"longBylineText": {
"runs": [
{
"text": "Kurzgesagt – In a Nutshell",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/user/Kurzgesagt",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl": "/user/Kurzgesagt",
},
},
},
],
},
}}.to_json,
# See first channelRenderer for detailed explanation. Besides channel data, the only difference
# between this channelRenderer and the previous one is the lack of an "subscriberCountText"
# as it is hidden on this channel.
{"channelRenderer": {
"channelId": "UCNhX3WQEkraW3VHPyup8jkQ",
"title": {
"simpleText": "Langfocus",
},
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCNhX3WQEkraW3VHPyup8jkQ",
"canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
},
},
"thumbnail": {
"thumbnails": [
{
"url": "//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s88-c-k-c0x00ffffff-no-rj-mo",
"width": 88,
"height": 88,
},
{
"url": "//yt3.ggpht.com/ytc/AKedOLRvsTYz7nlOWrGLc1GzlV96kXxY1Q9IE1KzqbXa3g=s176-c-k-c0x00ffffff-no-rj-mo",
"width": 176,
"height": 176,
},
],
},
"descriptionSnippet": {
"runs": [
{
"text": "Sharing my passion for languages and reaching out into the wider world.",
},
],
},
"shortBylineText": {
"runs": [
{
"text": "Langfocus",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCNhX3WQEkraW3VHPyup8jkQ",
"canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
},
},
},
],
},
"videoCountText": {
"runs": [
{
"text": "165",
},
{
"text": " videos",
},
],
},
# "subscriptionButton": {subscribed": false},
"ownerBadges": [
{
"metadataBadgeRenderer": {
"icon": {
"iconType": "CHECK_CIRCLE_THICK",
},
"style": "BADGE_STYLE_TYPE_VERIFIED",
"tooltip": "Verified",
"TrackingParams": "",
"accessibilityData": {
"label": "Verified",
},
},
},
],
# "subscribeButton": {...},
# "TrackingParams": "",
"longBylineText": {
"runs": [
{
"text": "Langfocus",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCNhX3WQEkraW3VHPyup8jkQ",
"canonicalBaseUrl": "/channel/UCNhX3WQEkraW3VHPyup8jkQ",
},
},
},
],
},
}}.to_json,
]

458
spec/item_jsons/playlist_renderer.cr Normal file
View file

@ -0,0 +1,458 @@
# The following are examples of InnerTube playlistRenderers
#
# A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself.
PLAYLIST_RENDERER_EXAMPLES = [
{"playlistRenderer": {
"playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
"title": {
"simpleText": "The Universe and Space stuff",
},
# Array of thumbnails in increasing quality, taken from the last few videos within the playlist.
"thumbnails": [
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg",
"width": 168,
"height": 94,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLBJlY_7z-Jfm-lPgZvzcLsuotYD2g",
"width": 196,
"height": 110,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCPYBEIoBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCollsqaYxSm_va6vSN6oK8mnSFhw",
"width": 246,
"height": 138,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDOCmzlwvvYsaaFO2u8lyWPrZULkw",
"width": 336,
"height": 188,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/G-WO-z-QuWI/default.jpg",
"width": 43,
"height": 20,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/qEfPBt9dU60/default.jpg",
"width": 43,
"height": 20,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/gLZJlf5rHVs/default.jpg",
"width": 43,
"height": 20,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/3mnSDifDSxQ/default.jpg",
"width": 43,
"height": 20,
},
],
},
],
# Amount of videos in playlist
"videoCount": "32",
# Endpoint to arrive on after clicking on renderer
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
"webPageType": "WEB_PAGE_TYPE_WATCH",
"rootVe": 3832,
},
},
"watchEndpoint": {
"videoId": "0FH9cgRhQ-k",
"playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
"params": "OAI%3D",
# "loggingContext": {...},
# "watchEndpointSupportedOnesieConfig": {...}
},
},
# Renderer for the view full playlist link. This is stored in a
# runs object inside
# "viewPlaylistText": {...},
# (short) Author information
"shortBylineText": {
"runs": [
{
"text": "Kurzgesagt – In a Nutshell",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/user/Kurzgesagt",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl": "/user/Kurzgesagt",
},
},
},
],
},
# Updated/Published date
"publishedTimeText": {
"simpleText": "Updated 7 days ago",
},
# Two or less videos from the playlist. This is used to render preview (text-only)
# next to the playlist on search results. Each content below is a mini videoRenderer
"videos": [
{
"childVideoRenderer": {
"title": {
"simpleText": "The Largest Black Hole in the Universe - Size Comparison",
},
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
"webPageType": "WEB_PAGE_TYPE_WATCH",
"rootVe": 3832,
},
},
"watchEndpoint": {
"videoId": "0FH9cgRhQ-k",
"playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
# "loggingContext": {...},
# "watchEndpointSupportedOnesieConfig": {...}
},
},
"lengthText": {
"accessibility": {
"accessibilityData": {
"label": "13 minutes, 44 seconds",
},
},
"simpleText": "13:44",
},
"videoId": "0FH9cgRhQ-k",
},
},
{
"childVideoRenderer": {
"title": {
"simpleText": "How To Terraform Venus (Quickly)",
},
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/watch?v=G-WO-z-QuWI&list=PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
"webPageType": "WEB_PAGE_TYPE_WATCH",
"rootVe": 3832,
},
},
"watchEndpoint": {
"videoId": "G-WO-z-QuWI",
"playlistId": "PLFs4vir_WsTwEd-nJgVJCZPNL3HALHHpF",
# "loggingContext": {...},
# "watchEndpointSupportedOnesieConfig": {...}
},
},
"lengthText": {
"accessibility": {
"accessibilityData": {
"label": "12 minutes, 48 seconds",
},
},
"simpleText": "12:48",
},
"videoId": "G-WO-z-QuWI",
},
},
],
# Amount of videos in playlist
"videoCountText": {
"runs": [
{
"text": "32",
},
{
"text": " videos",
},
],
},
# "TrackingParams": "",
# Overlay counting amount of videos in playlist
# "thumbnailText": {...},
# (Long) Author information
"longBylineText": {
"runs": [
{
"text": "Kurzgesagt – In a Nutshell",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/user/Kurzgesagt",
"webPageType": "WEB_PAGE_TYPE_CHANNEL",
"rootVe": 3611,
"apiUrl": "/youtubei/v1/browse",
},
},
"browseEndpoint": {
"browseId": "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl": "/user/Kurzgesagt",
},
},
},
],
},
# Owner badges
"ownerBadges": [
{
"metadataBadgeRenderer": {
"icon": {
"iconType": "CHECK_CIRCLE_THICK",
},
"style": "BADGE_STYLE_TYPE_VERIFIED",
"tooltip": "Verified",
"TrackingParams": "",
"accessibilityData": {
"label": "Verified",
},
},
},
],
# The actual thumbnail of the playlist
#
# YouTube allows for defining custom ones instead of just using the last video
# in the playlist. As such, that can be accessed from here.
"thumbnailRenderer": {
"playlistVideoThumbnailRenderer": {
"thumbnail": {
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLD9giG-6BICfsfD6p8l0OxjPEqiPg",
"width": 168,
"height": 94,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLBJlY_7z-Jfm-lPgZvzcLsuotYD2g",
"width": 196,
"height": 110,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCPYBEIoBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCollsqaYxSm_va6vSN6oK8mnSFhw",
"width": 246,
"height": 138,
},
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDOCmzlwvvYsaaFO2u8lyWPrZULkw",
"width": 336,
"height": 188,
},
],
},
},
},
# Thumbnail overlays such as the play all button or the video count.
# "thumbnailOverlays": []
}}.to_json,
# Playlists rendered on a grid has a slightly different format
#
# IE lack of author information
{"gridPlaylistRenderer": {
"playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs",
# Playlist thumbnail in ascending quality
"thumbnail": {
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg",
"width": 480,
"height": 270,
},
],
},
# Playlist title and endpoint it redirects to on click
"title": {
"runs": [
{
"text": "The Existential Crisis Playlist",
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs",
"webPageType": "WEB_PAGE_TYPE_WATCH",
"rootVe": 3832,
},
},
"watchEndpoint": {
"videoId": "0FH9cgRhQ-k",
"playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs",
"params": "OAI%3D",
# "loggingContext": {...},
# "watchEndpointSupportedOnesieConfig": {...}
},
},
},
],
},
# Video count text in format
"videoCountText": {
"runs": [
{
"text": "34",
},
{
"text": " videos",
},
],
},
# Endpoint to arrive on after clicking on renderer
"navigationEndpoint": {
"clickTrackingParams": "",
"commandMetadata": {
"webCommandMetadata": {
"url": "/watch?v=0FH9cgRhQ-k&list=PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs",
"webPageType": "WEB_PAGE_TYPE_WATCH",
"rootVe": 3832,
},
},
"watchEndpoint": {
"videoId": "0FH9cgRhQ-k",
"playlistId": "PLFs4vir_WsTxontcYm5ctqp89cNBJKNrs",
"params": "OAI%3D",
# "loggingContext": {...},
# "watchEndpointSupportedOnesieConfig": {...}
},
},
# Shortened video count text.
"videoCountShortText": {
"simpleText": "34",
},
# "TrackingParams": "",
# Array of thumbnails in increasing quality, taken from the last few videos within the playlist.
"sidebarThumbnails": [
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/JXeJANDKwDc/default.jpg",
"width": 43,
"height": 20,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/Jzfpyo-q-RM/default.jpg",
"width": 43,
"height": 20,
},
],
},
{
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/qEfPBt9dU60/default.jpg",
"width": 43,
"height": 20,
},
],
},
],
# Renderer for playlist size overlay on thumbnail
"thumbnailText": {
"runs": [
{
"text": "34",
"bold": true,
},
{
"text": " videos",
},
],
},
# Amount of badges the channel has. IE verified.
"ownerBadges": [
{
"metadataBadgeRenderer": {
"icon": {
"iconType": "CHECK_CIRCLE_THICK",
},
"style": "BADGE_STYLE_TYPE_VERIFIED",
"tooltip": "Verified",
"TrackingParams": "",
"accessibilityData": {
"label": "Verified",
},
},
},
],
# Playlist thumbnail in ascending quality
#
# TODO find difference between this and "thumbnail" object.
"thumbnailRenderer": {
"playlistVideoThumbnailRenderer": {
"thumbnail": {
"thumbnails": [
{
"url": "https://i.ytimg.com/vi/0FH9cgRhQ-k/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLD9depPKF_lMsYL7jWnLoCVyw-0pg",
"width": 480,
"height": 270,
},
],
},
},
},
# Thumbnail overlays such as the play all button or the video count.
# "thumbnailOverlays": [...],
# Renderer for the view full playlist link. This is stored in a
# runs object inside
# "viewPlaylistText": {...}
}}.to_json,
]

284
spec/item_jsons/video_renderer.cr Normal file
View file

@ -0,0 +1,284 @@
# The following are examples of InnerTube videoRenderers
#
# A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not**
# the watchable video itself.
VIDEO_RENDERER_EXAMPLES = [
{"videoRenderer" => {
# Video ID
"videoId" => "E1KkQrFEl2I",
# Array of thumbnails in increasing quality.
"thumbnail" => {
"thumbnails" => [
{
"url" => "https://i.ytimg.com/vi/E1KkQrFEl2I/hq720.jpg?sqp=-oaymwEjCOgCEMoBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLAE7cGsAxbjoQIKa04sXkfF9nTlzw",
"width" => 360,
"height" => 202,
},
{
"url" => "https://i.ytimg.com/vi/E1KkQrFEl2I/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBnetdf_Lj9C6XpUuIVDV0mn7B2ew",
"width" => 720,
"height" => 404,
},
],
},
# Title. Can also be simpleText
"title" => {
"runs" => [
{
"text" => "How Large Can a Bacteria get? Life & Size 3",
},
],
"accessibility" => {
"accessibilityData" => {
"label" => "How Large Can a Bacteria get? Life & Size 3 by Kurzgesagt – In a Nutshell 9 months ago 11 minutes, 5 seconds 7,324,534 views",
},
},
},
# (Long) Author information.
"longBylineText" => {
"runs" => [
{
"text" => "Kurzgesagt – In a Nutshell",
"navigationEndpoint" => {
"clickTrackingParams" => "",
"commandMetadata" => {
"webCommandMetadata" => {
"url" => "/user/Kurzgesagt",
"webPageType" => "WEB_PAGE_TYPE_CHANNEL",
"rootVe" => 3611,
"apiUrl" => "/youtubei/v1/browse",
},
},
"browseEndpoint" => {
"browseId" => "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl" => "/user/Kurzgesagt",
},
},
},
],
},
# Published date
#
# For live videos (and possibly recently premiered videos) there is no published information.
# Instead, in its place is the amount of people currently watching.
"publishedTimeText" => {
"simpleText" => "9 months ago",
},
# Video Length (locale specific?)
"lengthText" => {
"accessibility" => {
"accessibilityData" => {
"label" => "11 minutes, 5 seconds",
},
},
"simpleText" => "11:05",
},
# View count (locale specific?)
#
# Typically views are stored under a "simpleText" in the "viewCountText". However, for
# livestreams and premiered it is stored under a "runs" array: [{"text" =>123}, {"text" => "watching"}]
#
# When view count is disabled the "viewCountText" is not present on InnerTube data.
"viewCountText" => {
"simpleText" => "7,324,534 views",
},
# Endpoint to arrive on after clicking on renderer
"navigationEndpoint" => {
"clickTrackingParams" => "",
"commandMetadata" => {
"webCommandMetadata" => {
"url" => "/watch?v=E1KkQrFEl2I",
"webPageType" => "WEB_PAGE_TYPE_WATCH",
"rootVe" => 3832,
},
},
"watchEndpoint" => {
"videoId" => "E1KkQrFEl2I",
"params" => "qgMKa3Vyemdlc2FndLoDCgj0juTts8vxj1S6AwoI4Lz4wObkpqh0ugMLCLOGu73S_MCcuwG6AwoI6Nmyi6a2-8ZgugMKCImerNeaqIrWdroDHhIcUkRDTVVDc1hWazM3Ymx0SHhEMXJEUHd0Tk04UboDCgji8sL8s9_j8hu6AwoIroO-kuTyidxhugMLCJSWjb7iwfS03gG6AwsIrNvJxeruqv_0AboDCwiRr7iB0-b93uEBugMKCNTbv-uz04eIEroDCwii0dvbjej74ssBugMLCNbouPHd55iEjAG6AwoIpvvOyc3pwtVCugMLCJbux5v_3NKIrgG6AwsIrJPl67y13v6QAboDCwipjcSCw9Xl6qgB8gMFDSaq1Tw%3D",
"watchEndpointSupportedOnesieConfig" => {
"html5PlaybackOnesieConfig" => {
"commonConfig" => {
"url" => "https://r1---sn-nx57ynlk.googlevideo.com/initplayback?source=youtube&orc=1&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&msp=1&odeak=1&odepv=1&osfc=1&ip=198.54.131.169&id=1352a442b1449762&initcwndbps=2022500&mt=1628653969&oweuc=&pxtags=Cg4KAnR4EggyNDAyNzcwNg&rxtags=Cg4KAnR4EggyNDAyNzcwMw%2CCg4KAnR4EggyNDAyNzcwNA%2CCg4KAnR4EggyNDAyNzcwNQ%2CCg4KAnR4EggyNDAyNzcwNg",
},
},
},
},
},
# Video badges. IE Live, CC, etc
"badges" => [
{
"metadataBadgeRenderer" => {
"style" => "BADGE_STYLE_TYPE_SIMPLE",
"label" => "CC",
"trackingParams" => "",
"accessibilityData" => {
"label" => "Closed captions",
},
},
},
],
# Author badges
"ownerBadges" => [
{
"metadataBadgeRenderer" => {
"icon" => {
"iconType" => "CHECK_CIRCLE_THICK",
},
"style" => "BADGE_STYLE_TYPE_VERIFIED",
"tooltip" => "Verified",
"trackingParams" => "",
"accessibilityData" => {
"label" => "Verified",
},
},
},
],
# Author name
"ownerText" => {
"runs" => [
{
"text" => "Kurzgesagt – In a Nutshell",
"navigationEndpoint" => {
"clickTrackingParams" => "",
"commandMetadata" => {
"webCommandMetadata" => {
"url" => "/user/Kurzgesagt",
"webPageType" => "WEB_PAGE_TYPE_CHANNEL",
"rootVe" => 3611,
"apiUrl" => "/youtubei/v1/browse",
},
},
"browseEndpoint" => {
"browseId" => "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl" => "/user/Kurzgesagt",
},
},
},
],
},
# (Long) Author information.
# TODO find difference between short and long BylineText
"shortBylineText" => {
"runs" => [
{
"text" => "Kurzgesagt – In a Nutshell",
"navigationEndpoint" => {
"clickTrackingParams" => "",
"commandMetadata" => {
"webCommandMetadata" => {
"url" => "/user/Kurzgesagt",
"webPageType" => "WEB_PAGE_TYPE_CHANNEL",
"rootVe" => 3611,
"apiUrl" => "/youtubei/v1/browse",
},
},
"browseEndpoint" => {
"browseId" => "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl" => "/user/Kurzgesagt",
},
},
},
],
},
# "trackingParams" => "",
# "showActionMenu" => false,
"shortViewCountText" => {
"accessibility" => {
"accessibilityData" => {
"label" => "7.3 million views",
},
},
"simpleText" => "7.3M views",
},
# "menu" : {...} | renderer for 3 dot menu
# Channel pfp renderer. Also unused on Invidious
"channelThumbnailSupportedRenderers" => {
"channelThumbnailWithLinkRenderer" => {
"thumbnail" => {
"thumbnails" => [
{
"url" => "https://yt3.ggpht.com/ytc/AKedOLRvMf1ZTTCnC5Wc0EGOVPyrdyvfvs20vtdTUxz_vQ=s68-c-k-c0x00ffffff-no-rj",
"width" => 68,
"height" => 68,
},
],
},
"navigationEndpoint" => {
"clickTrackingParams" => "",
"commandMetadata" => {
"webCommandMetadata" => {
"url" => "/user/Kurzgesagt",
"webPageType" => "WEB_PAGE_TYPE_CHANNEL",
"rootVe" => 3611,
"apiUrl" => "/youtubei/v1/browse",
},
},
"browseEndpoint" => {
"browseId" => "UCsXVk37bltHxD1rDPwtNM8Q",
"canonicalBaseUrl" => "/user/Kurzgesagt",
},
},
"accessibility" => {
"accessibilityData" => {
"label" => "Go to channel",
},
},
},
},
# Provides the overlays on the thumbnails. This is currently
# used as an fallback for the "lengthText" attribute when that
# doesn't exist.
"thumbnailOverlays" => [
{
"thumbnailOverlayTimeStatusRenderer" => {
"text" => {
"accessibility" => {
"accessibilityData" => {
"label" => "11 minutes, 5 seconds",
},
},
"simpleText" => "11:05",
},
"style" => "DEFAULT",
},
},
# Renderer for watch later, add to playlist, etc overlay buttons on YouTube.
# Each separate btn has a different renderer
# {"thumbnailOverlayToggleButtonRenderer" => {...}}
# thumbnailOverlayNowPlayingRenderer: {...} | Renders "Now playing"
],
# Description snippet
"detailedMetadataSnippets" => [
{
"snippetText" => {
"runs" => [
{
"text" => "In and out, in and out. Staying alive is about doing things. This very second, your cells are combusting glucose molecules with ...",
},
],
},
"snippetHoverText" => {
"runs" => [
{
"text" => "From the video description",
},
],
},
"maxOneLine" => false,
},
],
}}.to_json,
]

12
src/invidious/data_structs/youtube/renderers/category.cr
View file

@ -18,14 +18,14 @@ module YouTubeStructs
property description_html : String
property badges : Array(Tuple(String, String))?
# Extracts all renderers out of the category's contents.
def extract_renderers()
target = [] of Renderer
# Extracts all renderers out of the category's contents.
def extract_renderers
target = [] of Renderer
@contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
return target
end
return target
end
def to_json(locale, json : JSON::Builder)
json.object do

2
src/invidious/data_structs/youtube/videos.cr
View file

@ -1,3 +1,5 @@
require "./caption"
module YouTubeStructs
# Converter to serialize first level JSON data as methods for the videos struct
module VideoJSONConverter

3
src/invidious/helpers/extractors.cr
View file

@ -66,6 +66,9 @@ private module Parsers
# TODO change default value to nil and typical encoding type to tuple storing type (watchers, views, etc)
# and count
view_count = item_contents.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
# TODO YouTube seems to have removed the description_html snippet and replaced it with "snippetText"
# inside the detailedMetadataSnippets attribute
description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
# The length information *should* only always exist in "lengthText". However, the legacy Invidious code

1
src/invidious/helpers/macros.cr
View file

@ -9,6 +9,7 @@ module DB::Serializable
}}
end
# Initialize DB::Serializable descendants via NamedTuples
def initialize(tuple)
\{% for var in @type.instance_vars %}
\{% ann = var.annotation(::DB::Field) %}