diff --git a/spec/invidious/videos/scheduled_live_extract_spec.cr b/spec/invidious/videos/scheduled_live_extract_spec.cr index 6e531bbd..ff5aacd5 100644 --- a/spec/invidious/videos/scheduled_live_extract_spec.cr +++ b/spec/invidious/videos/scheduled_live_extract_spec.cr @@ -1,6 +1,6 @@ require "../../parsers_helper.cr" -Spectator.describe Invidious::Hashtag do +Spectator.describe "parse_video_info" do it "parses scheduled livestreams data (test 1)" do # Enable mock _player = load_mock("video/scheduled_live_nintendo.player") @@ -12,26 +12,50 @@ Spectator.describe Invidious::Hashtag do # Some basic verifications expect(typeof(info)).to eq(Hash(String, JSON::Any)) - expect(info["shortDescription"].as_s).to eq( - "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." - ) - expect(info["descriptionHtml"].as_s).to eq( - "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." - ) + expect(info["videoType"].as_s).to eq("Scheduled") + # Basic video infos + + expect(info["title"].as_s).to eq("Xenoblade Chronicles 3 Nintendo Direct") + expect(info["views"].as_i).to eq(160) expect(info["likes"].as_i).to eq(2_283) + expect(info["lengthSeconds"].as_i).to eq(0_i64) + expect(info["published"].as_s).to eq("2022-06-22T14:00:00Z") # Unix 1655906400 - expect(info["genre"].as_s).to eq("Gaming") - expect(info["genreUrl"].raw).to be_nil - expect(info["genreUcid"].as_s).to be_empty - expect(info["license"].as_s).to be_empty + # Extra video infos - expect(info["authorThumbnail"].as_s).to eq( - "https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj" + expect(info["allowedRegions"].as_a).to_not be_empty + expect(info["allowedRegions"].as_a.size).to eq(249) + + expect(info["allowedRegions"].as_a).to contain( + "AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR", + "TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", + "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW" ) - expect(info["authorVerified"].as_bool).to be_true - expect(info["subCountText"].as_s).to eq("8.5M") + expect(info["keywords"].as_a).to_not be_empty + expect(info["keywords"].as_a.size).to eq(11) + + expect(info["keywords"].as_a).to contain_exactly( + "nintendo", + "game", + "gameplay", + "fun", + "video game", + "action", + "adventure", + "rpg", + "play", + "switch", + "nintendo switch" + ).in_any_order + + expect(info["allowRatings"].as_bool).to be_true + expect(info["isFamilyFriendly"].as_bool).to be_true + expect(info["isListed"].as_bool).to be_true + expect(info["isUpcoming"].as_bool).to be_true + + # Related videos expect(info["relatedVideos"].as_a.size).to eq(20) @@ -50,6 +74,32 @@ Spectator.describe Invidious::Hashtag do expect(info["relatedVideos"][16]["view_count"].as_s).to eq("53510") expect(info["relatedVideos"][16]["short_view_count"].as_s).to eq("53K") expect(info["relatedVideos"][16]["author_verified"].as_s).to eq("true") + + # Description + + description = "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." + + expect(info["description"].as_s).to eq(description) + expect(info["shortDescription"].as_s).to eq(description) + expect(info["descriptionHtml"].as_s).to eq(description) + + # Video metadata + + expect(info["genre"].as_s).to eq("Gaming") + expect(info["genreUcid"].as_s).to be_empty + expect(info["license"].as_s).to be_empty + + # Author infos + + expect(info["author"].as_s).to eq("Nintendo") + expect(info["ucid"].as_s).to eq("UCGIY_O-8vW4rfX98KlMkvRg") + + expect(info["authorThumbnail"].as_s).to eq( + "https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj" + ) + + expect(info["authorVerified"].as_bool).to be_true + expect(info["subCountText"].as_s).to eq("8.5M") end it "parses scheduled livestreams data (test 2)" do @@ -63,34 +113,63 @@ Spectator.describe Invidious::Hashtag do # Some basic verifications expect(typeof(info)).to eq(Hash(String, JSON::Any)) - expect(info["shortDescription"].as_s).to start_with( - <<-TXT - PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. + expect(info["videoType"].as_s).to eq("Scheduled") - Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL - TXT - ) - expect(info["descriptionHtml"].as_s).to start_with( - <<-TXT - PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. - - Join the channel to get exclusive access to perks: bit.ly/3Q9rSQL - TXT - ) + # Basic video infos + expect(info["title"].as_s).to eq("The Truth About Greenpeace w/ Dr. Patrick Moore | PBD Podcast | Ep. 171") + expect(info["views"].as_i).to eq(24) expect(info["likes"].as_i).to eq(22) + expect(info["lengthSeconds"].as_i).to eq(0_i64) + expect(info["published"].as_s).to eq("2022-07-14T13:00:00Z") # Unix 1657803600 - expect(info["genre"].as_s).to eq("Entertainment") - expect(info["genreUrl"].raw).to be_nil - expect(info["genreUcid"].as_s).to be_empty - expect(info["license"].as_s).to be_empty + # Extra video infos - expect(info["authorThumbnail"].as_s).to eq( - "https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj" + expect(info["allowedRegions"].as_a).to_not be_empty + expect(info["allowedRegions"].as_a.size).to eq(249) + + expect(info["allowedRegions"].as_a).to contain( + "AD", "AR", "BA", "BT", "CZ", "FO", "GL", "IO", "KE", "KH", "LS", + "LT", "MP", "NO", "PR", "RO", "SE", "SK", "SS", "SX", "SZ", "ZW" ) - expect(info["authorVerified"].as_bool).to be_false - expect(info["subCountText"].as_s).to eq("227K") + expect(info["keywords"].as_a).to_not be_empty + expect(info["keywords"].as_a.size).to eq(25) + + expect(info["keywords"].as_a).to contain_exactly( + "Patrick Bet-David", + "Valeutainment", + "The BetDavid Podcast", + "The BetDavid Show", + "Betdavid", + "PBD", + "BetDavid show", + "Betdavid podcast", + "podcast betdavid", + "podcast patrick", + "patrick bet david podcast", + "Valuetainment podcast", + "Entrepreneurs", + "Entrepreneurship", + "Entrepreneur Motivation", + "Entrepreneur Advice", + "Startup Entrepreneurs", + "valuetainment", + "patrick bet david", + "PBD podcast", + "Betdavid show", + "Betdavid Podcast", + "Podcast Betdavid", + "Show Betdavid", + "PBDPodcast" + ).in_any_order + + expect(info["allowRatings"].as_bool).to be_true + expect(info["isFamilyFriendly"].as_bool).to be_true + expect(info["isListed"].as_bool).to be_true + expect(info["isUpcoming"].as_bool).to be_true + + # Related videos expect(info["relatedVideos"].as_a.size).to eq(20) @@ -109,5 +188,41 @@ Spectator.describe Invidious::Hashtag do expect(info["relatedVideos"][9]["view_count"]).to eq("26432") expect(info["relatedVideos"][9]["short_view_count"]).to eq("26K") expect(info["relatedVideos"][9]["author_verified"]).to eq("true") + + # Description + + description_start_text = <<-TXT + PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. + + Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL + TXT + + expect(info["description"].as_s).to start_with(description_start_text) + expect(info["shortDescription"].as_s).to start_with(description_start_text) + + expect(info["descriptionHtml"].as_s).to start_with( + <<-TXT + PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. + + Join the channel to get exclusive access to perks: bit.ly/3Q9rSQL + TXT + ) + + # Video metadata + + expect(info["genre"].as_s).to eq("Entertainment") + expect(info["genreUcid"].as_s).to be_empty + expect(info["license"].as_s).to be_empty + + # Author infos + + expect(info["author"].as_s).to eq("PBD Podcast") + expect(info["ucid"].as_s).to eq("UCGX7nGXpz-CmO_Arg-cgJ7A") + + expect(info["authorThumbnail"].as_s).to eq( + "https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj" + ) + expect(info["authorVerified"].as_bool).to be_false + expect(info["subCountText"].as_s).to eq("227K") end end diff --git a/src/invidious.cr b/src/invidious.cr index 070b4d18..464906ed 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -37,6 +37,7 @@ require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" +require "./invidious/videos/*" require "./invidious/*" require "./invidious/channels/*" diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 80b67641..a9b00860 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage getter full_videos : Array(Hash(String, JSON::Any)) getter video_streams : Array(Hash(String, JSON::Any)) getter audio_streams : Array(Hash(String, JSON::Any)) - getter captions : Array(Caption) + getter captions : Array(Invidious::Videos::Caption) def initialize( @full_videos, @@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage video_assets.full_videos.each do |option| mimetype = option["mimeType"].as_s.split(";")[0] - height = itag_to_metadata?(option["itag"]).try &.["height"]? + height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]? value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index f87c6b47..f1688581 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1,275 +1,12 @@ -CAPTION_LANGUAGES = { - "", - "English", - "English (auto-generated)", - "English (United Kingdom)", - "English (United States)", - "Afrikaans", - "Albanian", - "Amharic", - "Arabic", - "Armenian", - "Azerbaijani", - "Bangla", - "Basque", - "Belarusian", - "Bosnian", - "Bulgarian", - "Burmese", - "Cantonese (Hong Kong)", - "Catalan", - "Cebuano", - "Chinese", - "Chinese (China)", - "Chinese (Hong Kong)", - "Chinese (Simplified)", - "Chinese (Taiwan)", - "Chinese (Traditional)", - "Corsican", - "Croatian", - "Czech", - "Danish", - "Dutch", - "Dutch (auto-generated)", - "Esperanto", - "Estonian", - "Filipino", - "Finnish", - "French", - "French (auto-generated)", - "Galician", - "Georgian", - "German", - "German (auto-generated)", - "Greek", - "Gujarati", - "Haitian Creole", - "Hausa", - "Hawaiian", - "Hebrew", - "Hindi", - "Hmong", - "Hungarian", - "Icelandic", - "Igbo", - "Indonesian", - "Indonesian (auto-generated)", - "Interlingue", - "Irish", - "Italian", - "Italian (auto-generated)", - "Japanese", - "Japanese (auto-generated)", - "Javanese", - "Kannada", - "Kazakh", - "Khmer", - "Korean", - "Korean (auto-generated)", - "Kurdish", - "Kyrgyz", - "Lao", - "Latin", - "Latvian", - "Lithuanian", - "Luxembourgish", - "Macedonian", - "Malagasy", - "Malay", - "Malayalam", - "Maltese", - "Maori", - "Marathi", - "Mongolian", - "Nepali", - "Norwegian Bokmål", - "Nyanja", - "Pashto", - "Persian", - "Polish", - "Portuguese", - "Portuguese (auto-generated)", - "Portuguese (Brazil)", - "Punjabi", - "Romanian", - "Russian", - "Russian (auto-generated)", - "Samoan", - "Scottish Gaelic", - "Serbian", - "Shona", - "Sindhi", - "Sinhala", - "Slovak", - "Slovenian", - "Somali", - "Southern Sotho", - "Spanish", - "Spanish (auto-generated)", - "Spanish (Latin America)", - "Spanish (Mexico)", - "Spanish (Spain)", - "Sundanese", - "Swahili", - "Swedish", - "Tajik", - "Tamil", - "Telugu", - "Thai", - "Turkish", - "Turkish (auto-generated)", - "Ukrainian", - "Urdu", - "Uzbek", - "Vietnamese", - "Vietnamese (auto-generated)", - "Welsh", - "Western Frisian", - "Xhosa", - "Yiddish", - "Yoruba", - "Zulu", -} +require "./videos/caption" +require "./videos/formats" REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} -# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 -VIDEO_FORMATS = { - "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, - "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, - "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, - "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, - "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, - "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - - # 3D videos - "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, - "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, - "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, - - # Apple HTTP Live Streaming - "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, - "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, - "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, - "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, - - # DASH mp4 video - "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, - "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, - "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, - "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, - "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) - "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, - "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, - "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, - "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, - "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, - - # Dash mp4 audio - "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, - "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, - "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, - "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, - "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, - "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, - - # Dash webm - "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, - "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, - "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, - "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, - "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, - "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, - "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, - "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, - # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, - "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, - - # Dash webm audio - "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, - "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, - - # Dash webm audio with opus inside - "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, - "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, - "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, - - # av01 video only formats sometimes served with "unknown" codecs - "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, - "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, - "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, - "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, -} - -struct VideoPreferences - include JSON::Serializable - - property annotations : Bool - property autoplay : Bool - property comments : Array(String) - property continue : Bool - property continue_autoplay : Bool - property controls : Bool - property listen : Bool - property local : Bool - property preferred_captions : Array(String) - property player_style : String - property quality : String - property quality_dash : String - property raw : Bool - property region : String? - property related_videos : Bool - property speed : Float32 | Float64 - property video_end : Float64 | Int32 - property video_loop : Bool - property extend_desc : Bool - property video_start : Float64 | Int32 - property volume : Int32 - property vr_mode : Bool - property save_player_pos : Bool +enum VideoType + Video + Livestream + Scheduled end struct Video @@ -282,7 +19,7 @@ struct Video property updated : Time @[DB::Field(ignore: true)] - property captions : Array(Caption)? + property captions : Array(Invidious::Videos::Caption)? @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? @@ -301,7 +38,7 @@ struct Video def to_json(locale : String?, json : JSON::Builder) json.object do - json.field "type", "video" + json.field "type", self.video_type json.field "title", self.title json.field "videoId", self.id @@ -392,7 +129,7 @@ struct Video json.field "lmt", fmt["lastModified"] json.field "projectionType", fmt["projectionType"] - if fmt_info = itag_to_metadata?(fmt["itag"]) + if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps json.field "container", fmt_info["ext"] @@ -439,7 +176,7 @@ struct Video json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] - fmt_info = itag_to_metadata?(fmt["itag"]) + fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) if fmt_info fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps @@ -527,61 +264,21 @@ struct Video to_json(nil, json) end - def title - info["videoDetails"]["title"]?.try &.as_s || "" - end - - def ucid - info["videoDetails"]["channelId"]?.try &.as_s || "" - end - - def author - info["videoDetails"]["author"]?.try &.as_s || "" - end - - def length_seconds : Int32 - info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i || - info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 - end - - def views : Int64 - info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 - end - - def likes : Int64 - info["likes"]?.try &.as_i64 || 0_i64 - end - - def dislikes : Int64 - info["dislikes"]?.try &.as_i64 || 0_i64 + def video_type : VideoType + return VideoType.parse(info["videoType"]?.try &.as_s || "regular") end def published : Time - info - .dig?("microformat", "playerMicroformatRenderer", "publishDate") + return info["published"]? .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) - info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) - end - - def allow_ratings - r = info["videoDetails"]["allowRatings"]?.try &.as_bool - r.nil? ? false : r + info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end def live_now - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false - end - - def is_listed - info["videoDetails"]["isCrawlable"]?.try &.as_bool || false - end - - def is_upcoming - info["videoDetails"]["isUpcoming"]?.try &.as_bool || false + return (self.video_type == VideoType::Livestream) end def premiere_timestamp : Time? @@ -590,31 +287,11 @@ struct Video .try { |t| Time.parse_rfc3339(t.as_s) } end - def keywords - info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String - end - def related_videos info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) end - def allowed_regions - info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - end - - def author_thumbnail : String - info["authorThumbnail"]?.try &.as_s || "" - end - - def author_verified : Bool - info["authorVerified"]?.try &.as_bool || false - end - - def sub_count_text : String - info["subCountText"]?.try &.as_s || "-" - end + # Methods for parsing streaming data def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream @@ -665,6 +342,8 @@ struct Video adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") end + # Misc. methods + def storyboards storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") .try &.as_s.split("|") @@ -728,51 +407,21 @@ struct Video end def paid - reason = info.dig?("playabilityStatus", "reason").try &.as_s || "" - return reason.includes? "requires payment" + return (self.reason || "").includes? "requires payment" end def premium keywords.includes? "YouTube Red" end - def captions : Array(Caption) - return @captions.as(Array(Caption)) if @captions - captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] - language_code = caption["languageCode"].to_s - base_url = caption["baseUrl"].to_s - - caption = Caption.new(name.to_s, language_code, base_url) - caption.name = caption.name.split(" - ")[0] - caption + def captions : Array(Invidious::Videos::Caption) + if @captions + return @captions.as(Array(Invidious::Videos::Caption)) + else + captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) + @captions = captions + return captions end - captions ||= [] of Caption - @captions = captions - return @captions.as(Array(Caption)) - end - - def description - description = info - .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") - .try &.as_s || "" - end - - # TODO - def description=(value : String) - @description = value - end - - def description_html - info["descriptionHtml"]?.try &.as_s || "

" - end - - def description_html=(value : String) - info["descriptionHtml"] = JSON::Any.new(value) - end - - def short_description - info["shortDescription"]?.try &.as_s? || "" end def hls_manifest_url : String? @@ -783,25 +432,12 @@ struct Video info.dig?("streamingData", "dashManifestUrl").try &.as_s end - def genre : String - info["genre"]?.try &.as_s || "" - end - def genre_url : String? info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil end - def license : String? - info["license"]?.try &.as_s - end - - def is_family_friendly : Bool - info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false - end - def is_vr : Bool? - projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s - return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type + return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end def projection_type : String? @@ -811,21 +447,91 @@ struct Video def reason : String? info["reason"]?.try &.as_s end -end -struct Caption - property name - property language_code - property base_url + # Macros defining getters/setters for various types of data - getter name : String - getter language_code : String - getter base_url : String + private macro getset_string(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : String + return info[{{name.stringify}}]?.try &.as_s || "" + end - setter name + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : String) + info[{{name.stringify}}] = JSON::Any.new(value) + end - def initialize(@name, @language_code, @base_url) + {% if flag?(:debug_macros) %} {{debug}} {% end %} end + + private macro getset_string_array(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Array(String) + return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String + end + + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Array(String)) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + + {% for op, type in {i32: Int32, i64: Int64} %} + private macro getset_{{op}}(name) + def \{{name.id.underscore}} : {{type}} + return info[\{{name.stringify}}]?.try &.as_i.to_{{op}} || 0_{{op}} + end + + def \{{name.id.underscore}}=(value : Int) + info[\{{name.stringify}}] = JSON::Any.new(value.to_i64) + end + + \{% if flag?(:debug_macros) %} \{{debug}} \{% end %} + end + {% end %} + + private macro getset_bool(name) + # Return {{name.stringify}} from `info` + def {{name.id.underscore}} : Bool + return info[{{name.stringify}}]?.try &.as_bool || false + end + + # Update {{name.stringify}} into `info` + def {{name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + + # Method definitions, using the macros above + + getset_string author + getset_string authorThumbnail + getset_string description + getset_string descriptionHtml + getset_string genre + getset_string genreUcid + getset_string license + getset_string shortDescription + getset_string subCountText + getset_string title + getset_string ucid + + getset_string_array allowedRegions + getset_string_array keywords + + getset_i32 lengthSeconds + getset_i64 likes + getset_i64 views + + getset_bool allowRatings + getset_bool authorVerified + getset_bool isFamilyFriendly + getset_bool isListed + getset_bool isUpcoming end class VideoRedirect < Exception @@ -973,6 +679,46 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer + video_details = player_response.dig?("videoDetails") + microformat = player_response.dig?("microformat", "playerMicroformatRenderer") + + raise BrokenTubeException.new("videoDetails") if !video_details + raise BrokenTubeException.new("microformat") if !microformat + + # Basic video infos + + title = video_details["title"]?.try &.as_s + + views = video_primary_renderer + .dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") + .try &.as_s.to_i64 + views ||= video_details["viewCount"]?.try &.as_s.to_i64 + + length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) + .try &.as_s.to_i64 + + published = microformat["publishDate"]? + .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + + premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") + .try { |t| Time.parse_rfc3339(t.as_s) } + + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") + .try &.as_bool || false + + # Extra video infos + + allowed_regions = microformat["availableCountries"]? + .try &.as_a.map &.as_s || [] of String + + allow_ratings = video_details["allowRatings"]?.try &.as_bool + family_friendly = microformat["isFamilySafe"].try &.as_bool + is_listed = video_details["isCrawlable"]?.try &.as_bool + is_upcoming = video_details["isUpcoming"]?.try &.as_bool + + keywords = video_details["keywords"]? + .try &.as_a.map &.as_s || [] of String + # Related videos LOGGER.debug("extract_video_info: parsing related videos...") @@ -1027,6 +773,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Description + description = microformat.dig?("description", "simpleText").try &.as_s || "" short_description = player_response.dig?("videoDetails", "shortDescription") description_html = video_secondary_renderer.try &.dig?("description", "runs") @@ -1038,7 +785,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") .try &.as_a - genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category") + genre = microformat["category"]? genre_ucid = nil license = nil @@ -1060,6 +807,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Author infos + author = video_details["author"]?.try &.as_s + ucid = video_details["channelId"]?.try &.as_s + if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") author_verified = has_verified_badge?(author_info["badges"]?) @@ -1071,19 +821,46 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Return data + if live_now + video_type = VideoType::Livestream + elsif premiere_timestamp.not_nil! + video_type = VideoType::Scheduled + published = premiere_timestamp || Time.utc + else + video_type = VideoType::Video + end + params = { - "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), - "relatedVideos" => JSON::Any.new(related), - "likes" => JSON::Any.new(likes || 0_i64), - "dislikes" => JSON::Any.new(0_i64), + "videoType" => JSON::Any.new(video_type.to_s), + # Basic video infos + "title" => JSON::Any.new(title || ""), + "views" => JSON::Any.new(views || 0_i64), + "likes" => JSON::Any.new(likes || 0_i64), + "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), + "published" => JSON::Any.new(published.to_rfc3339), + # Extra video infos + "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), + "allowRatings" => JSON::Any.new(allow_ratings || false), + "isFamilyFriendly" => JSON::Any.new(family_friendly || false), + "isListed" => JSON::Any.new(is_listed || false), + "isUpcoming" => JSON::Any.new(is_upcoming || false), + "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + # Related videos + "relatedVideos" => JSON::Any.new(related), + # Description + "description" => JSON::Any.new(description || ""), "descriptionHtml" => JSON::Any.new(description_html || "

"), - "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUrl" => JSON::Any.new(nil), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), - "license" => JSON::Any.new(license.try &.as_s || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified), - "subCountText" => JSON::Any.new(subs_text || "-"), + "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + # Video metadata + "genre" => JSON::Any.new(genre.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), + "license" => JSON::Any.new(license.try &.as_s || ""), + # Author infos + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified), + "subCountText" => JSON::Any.new(subs_text || "-"), } return params @@ -1158,10 +935,6 @@ def fetch_video(id, region) return video end -def itag_to_metadata?(itag : JSON::Any) - return VIDEO_FORMATS[itag.to_s]? -end - def process_continuation(query, plid, id) continuation = nil if plid @@ -1176,135 +949,6 @@ def process_continuation(query, plid, id) continuation end -def process_video_params(query, preferences) - annotations = query["iv_load_policy"]?.try &.to_i? - autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - comments = query["comments"]?.try &.split(",").map(&.downcase) - continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } - continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } - local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } - player_style = query["player_style"]? - preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) - quality = query["quality"]? - quality_dash = query["quality_dash"]? - region = query["region"]? - related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - speed = query["speed"]?.try &.rchop("x").to_f? - video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } - extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } - volume = query["volume"]?.try &.to_i? - vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } - save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - if preferences - # region ||= preferences.region - annotations ||= preferences.annotations.to_unsafe - autoplay ||= preferences.autoplay.to_unsafe - comments ||= preferences.comments - continue ||= preferences.continue.to_unsafe - continue_autoplay ||= preferences.continue_autoplay.to_unsafe - listen ||= preferences.listen.to_unsafe - local ||= preferences.local.to_unsafe - player_style ||= preferences.player_style - preferred_captions ||= preferences.captions - quality ||= preferences.quality - quality_dash ||= preferences.quality_dash - related_videos ||= preferences.related_videos.to_unsafe - speed ||= preferences.speed - video_loop ||= preferences.video_loop.to_unsafe - extend_desc ||= preferences.extend_desc.to_unsafe - volume ||= preferences.volume - vr_mode ||= preferences.vr_mode.to_unsafe - save_player_pos ||= preferences.save_player_pos.to_unsafe - end - - annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe - autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe - comments ||= CONFIG.default_user_preferences.comments - continue ||= CONFIG.default_user_preferences.continue.to_unsafe - continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe - listen ||= CONFIG.default_user_preferences.listen.to_unsafe - local ||= CONFIG.default_user_preferences.local.to_unsafe - player_style ||= CONFIG.default_user_preferences.player_style - preferred_captions ||= CONFIG.default_user_preferences.captions - quality ||= CONFIG.default_user_preferences.quality - quality_dash ||= CONFIG.default_user_preferences.quality_dash - related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe - speed ||= CONFIG.default_user_preferences.speed - video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe - extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe - volume ||= CONFIG.default_user_preferences.volume - vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe - save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe - - annotations = annotations == 1 - autoplay = autoplay == 1 - continue = continue == 1 - continue_autoplay = continue_autoplay == 1 - listen = listen == 1 - local = local == 1 - related_videos = related_videos == 1 - video_loop = video_loop == 1 - extend_desc = extend_desc == 1 - vr_mode = vr_mode == 1 - save_player_pos = save_player_pos == 1 - - if CONFIG.disabled?("dash") && quality == "dash" - quality = "high" - end - - if CONFIG.disabled?("local") && local - local = false - end - - if start = query["t"]? || query["time_continue"]? || query["start"]? - video_start = decode_time(start) - end - video_start ||= 0 - - if query["end"]? - video_end = decode_time(query["end"]) - end - video_end ||= -1 - - raw = query["raw"]?.try &.to_i? - raw ||= 0 - raw = raw == 1 - - controls = query["controls"]?.try &.to_i? - controls ||= 1 - controls = controls >= 1 - - params = VideoPreferences.new({ - annotations: annotations, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, - preferred_captions: preferred_captions, - quality: quality, - quality_dash: quality_dash, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - extend_desc: extend_desc, - video_start: video_start, - volume: volume, - vr_mode: vr_mode, - save_player_pos: save_player_pos, - }) - - return params -end - def build_thumbnails(id) return { {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr new file mode 100644 index 00000000..4642c1a7 --- /dev/null +++ b/src/invidious/videos/caption.cr @@ -0,0 +1,168 @@ +require "json" + +module Invidious::Videos + struct Caption + property name : String + property language_code : String + property base_url : String + + def initialize(@name, @language_code, @base_url) + end + + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any) : Array(Caption) + caption_tracks = container + .dig?("playerCaptionsTracklistRenderer", "captionTracks") + .try &.as_a + + captions_list = [] of Caption + return captions_list if caption_tracks.nil? + + caption_tracks.each do |caption| + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + name = name.to_s.split(" - ")[0] + + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s + + captions_list << Caption.new(name, language_code, base_url) + end + + return captions_list + end + + # List of all caption languages available on Youtube. + LANGUAGES = { + "", + "English", + "English (auto-generated)", + "English (United Kingdom)", + "English (United States)", + "Afrikaans", + "Albanian", + "Amharic", + "Arabic", + "Armenian", + "Azerbaijani", + "Bangla", + "Basque", + "Belarusian", + "Bosnian", + "Bulgarian", + "Burmese", + "Cantonese (Hong Kong)", + "Catalan", + "Cebuano", + "Chinese", + "Chinese (China)", + "Chinese (Hong Kong)", + "Chinese (Simplified)", + "Chinese (Taiwan)", + "Chinese (Traditional)", + "Corsican", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Dutch (auto-generated)", + "Esperanto", + "Estonian", + "Filipino", + "Finnish", + "French", + "French (auto-generated)", + "Galician", + "Georgian", + "German", + "German (auto-generated)", + "Greek", + "Gujarati", + "Haitian Creole", + "Hausa", + "Hawaiian", + "Hebrew", + "Hindi", + "Hmong", + "Hungarian", + "Icelandic", + "Igbo", + "Indonesian", + "Indonesian (auto-generated)", + "Interlingue", + "Irish", + "Italian", + "Italian (auto-generated)", + "Japanese", + "Japanese (auto-generated)", + "Javanese", + "Kannada", + "Kazakh", + "Khmer", + "Korean", + "Korean (auto-generated)", + "Kurdish", + "Kyrgyz", + "Lao", + "Latin", + "Latvian", + "Lithuanian", + "Luxembourgish", + "Macedonian", + "Malagasy", + "Malay", + "Malayalam", + "Maltese", + "Maori", + "Marathi", + "Mongolian", + "Nepali", + "Norwegian Bokmål", + "Nyanja", + "Pashto", + "Persian", + "Polish", + "Portuguese", + "Portuguese (auto-generated)", + "Portuguese (Brazil)", + "Punjabi", + "Romanian", + "Russian", + "Russian (auto-generated)", + "Samoan", + "Scottish Gaelic", + "Serbian", + "Shona", + "Sindhi", + "Sinhala", + "Slovak", + "Slovenian", + "Somali", + "Southern Sotho", + "Spanish", + "Spanish (auto-generated)", + "Spanish (Latin America)", + "Spanish (Mexico)", + "Spanish (Spain)", + "Sundanese", + "Swahili", + "Swedish", + "Tajik", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Turkish (auto-generated)", + "Ukrainian", + "Urdu", + "Uzbek", + "Vietnamese", + "Vietnamese (auto-generated)", + "Welsh", + "Western Frisian", + "Xhosa", + "Yiddish", + "Yoruba", + "Zulu", + } + end +end diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr new file mode 100644 index 00000000..e98e7257 --- /dev/null +++ b/src/invidious/videos/formats.cr @@ -0,0 +1,116 @@ +module Invidious::Videos::Formats + def self.itag_to_metadata?(itag : JSON::Any) + return FORMATS[itag.to_s]? + end + + # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 + private FORMATS = { + "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, + "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, + "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, + "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, + "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + # 3D videos + "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + + # Apple HTTP Live Streaming + "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, + + # DASH mp4 video + "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, + "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, + "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, + "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, + "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) + "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, + "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, + "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, + + # Dash mp4 audio + "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, + "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, + "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, + "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, + "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, + + # Dash webm + "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, + "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, + "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, + "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, + "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, + "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, + # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + + # Dash webm audio + "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, + "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, + + # Dash webm audio with opus inside + "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, + "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, + "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, + + # av01 video only formats sometimes served with "unknown" codecs + "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, + "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, + "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, + "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, + } +end diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr new file mode 100644 index 00000000..34cf7ff0 --- /dev/null +++ b/src/invidious/videos/video_preferences.cr @@ -0,0 +1,156 @@ +struct VideoPreferences + include JSON::Serializable + + property annotations : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property quality_dash : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property extend_desc : Bool + property video_start : Float64 | Int32 + property volume : Int32 + property vr_mode : Bool + property save_player_pos : Bool +end + +def process_video_params(query, preferences) + annotations = query["iv_load_policy"]?.try &.to_i? + autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + comments = query["comments"]?.try &.split(",").map(&.downcase) + continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } + continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } + local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } + player_style = query["player_style"]? + preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) + quality = query["quality"]? + quality_dash = query["quality_dash"]? + region = query["region"]? + related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + speed = query["speed"]?.try &.rchop("x").to_f? + video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } + extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } + volume = query["volume"]?.try &.to_i? + vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } + save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + if preferences + # region ||= preferences.region + annotations ||= preferences.annotations.to_unsafe + autoplay ||= preferences.autoplay.to_unsafe + comments ||= preferences.comments + continue ||= preferences.continue.to_unsafe + continue_autoplay ||= preferences.continue_autoplay.to_unsafe + listen ||= preferences.listen.to_unsafe + local ||= preferences.local.to_unsafe + player_style ||= preferences.player_style + preferred_captions ||= preferences.captions + quality ||= preferences.quality + quality_dash ||= preferences.quality_dash + related_videos ||= preferences.related_videos.to_unsafe + speed ||= preferences.speed + video_loop ||= preferences.video_loop.to_unsafe + extend_desc ||= preferences.extend_desc.to_unsafe + volume ||= preferences.volume + vr_mode ||= preferences.vr_mode.to_unsafe + save_player_pos ||= preferences.save_player_pos.to_unsafe + end + + annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe + comments ||= CONFIG.default_user_preferences.comments + continue ||= CONFIG.default_user_preferences.continue.to_unsafe + continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe + listen ||= CONFIG.default_user_preferences.listen.to_unsafe + local ||= CONFIG.default_user_preferences.local.to_unsafe + player_style ||= CONFIG.default_user_preferences.player_style + preferred_captions ||= CONFIG.default_user_preferences.captions + quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash + related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe + speed ||= CONFIG.default_user_preferences.speed + video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe + extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe + volume ||= CONFIG.default_user_preferences.volume + vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe + save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe + + annotations = annotations == 1 + autoplay = autoplay == 1 + continue = continue == 1 + continue_autoplay = continue_autoplay == 1 + listen = listen == 1 + local = local == 1 + related_videos = related_videos == 1 + video_loop = video_loop == 1 + extend_desc = extend_desc == 1 + vr_mode = vr_mode == 1 + save_player_pos = save_player_pos == 1 + + if CONFIG.disabled?("dash") && quality == "dash" + quality = "high" + end + + if CONFIG.disabled?("local") && local + local = false + end + + if start = query["t"]? || query["time_continue"]? || query["start"]? + video_start = decode_time(start) + end + video_start ||= 0 + + if query["end"]? + video_end = decode_time(query["end"]) + end + video_end ||= -1 + + raw = query["raw"]?.try &.to_i? + raw ||= 0 + raw = raw == 1 + + controls = query["controls"]?.try &.to_i? + controls ||= 1 + controls = controls >= 1 + + params = VideoPreferences.new({ + annotations: annotations, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, + preferred_captions: preferred_captions, + quality: quality, + quality_dash: quality_dash, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + extend_desc: extend_desc, + video_start: video_start, + volume: volume, + vr_mode: vr_mode, + save_player_pos: save_player_pos, + }) + + return params +end diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index dbb5e9db..d841982c 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -89,7 +89,7 @@ <% preferences.captions.each_with_index do |caption, index| %>