iv-org-mirror/invidious-copy-2023-06-08
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2023-06-08.git synced 2024-08-15 00:53:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
Wes van der Vleuten 2023-01-21 23:35:39 +01:00
commit 420e12bb8b
113 changed files with 3023 additions and 2338 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/invidious.cr
View file

@ -34,9 +34,13 @@ require "protodec/utils"
require "./invidious/database/*"
require "./invidious/database/migrations/*"
require "./invidious/http_server/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
require "./invidious/frontend/*"
require "./invidious/videos/*"
require "./invidious/jsonify/**"
require "./invidious/*"
require "./invidious/channels/*"
@ -45,6 +49,13 @@ require "./invidious/search/*"
require "./invidious/routes/**"
require "./invidious/jobs/**"
# Declare the base namespace for invidious
module Invidious
end
# Simple alias to make code easier to read
alias IV = Invidious
CONFIG = Config.load
HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
@ -169,7 +180,7 @@ if CONFIG.popular_enabled
Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
end
CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new

119
src/invidious/channels/about.cr
View file

@ -16,12 +16,6 @@ record AboutChannel,
tabs : Array(String),
verified : Bool
record AboutRelatedChannel,
ucid : String,
author : String,
author_url : String,
author_thumbnail : String
def get_about_info(ucid, locale) : AboutChannel
begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@ -100,38 +94,51 @@ def get_about_info(ucid, locale) : AboutChannel
total_views = 0_i64
joined = Time.unix(0)
tabs = [] of String
tab_names = [] of String
tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
if !tabs_json.nil?
# Retrieve information from the tabs array. The index we are looking for varies between channels.
tabs_json.each do |node|
# Try to find the about section which is located in only one of the tabs.
channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
.try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
.try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
if !channel_about_meta.nil?
total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
(channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
auto_generated = true
end
end
# This is a small fix to not add extra code on the HTML side
# I.e, the URL for the "live" tab is .../streams, so use "streams"
# everywhere for the sake of simplicity
(name == "live") ? "streams" : name
end
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
# Try to find the about metadata section
channel_about_meta = about_tab.dig?(
"content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"channelAboutFullMetadataRenderer"
)
if !channel_about_meta.nil?
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
)
end
tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
end
sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0
sub_count = initdata
.dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0
AboutChannel.new(
ucid: ucid,
@ -147,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel
joined: joined,
is_family_friendly: is_family_friendly,
allowed_regions: allowed_regions,
tabs: tabs,
tabs: tab_names,
verified: author_verified || false,
)
end
def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
# params is {"2:string":"channels"} encoded
channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
return [] of AboutRelatedChannel if tab.nil?
items = tab.dig?(
"tabRenderer", "content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"gridRenderer", "items"
).try &.as_a?
related = [] of AboutRelatedChannel
return related if (items.nil? || items.empty?)
items.each do |item|
renderer = item["gridChannelRenderer"]?
next if !renderer
related_id = renderer.dig("channelId").as_s
related_title = renderer.dig("title", "simpleText").as_s
related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
related << AboutRelatedChannel.new(
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
)
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
return related
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
end

81
src/invidious/channels/channels.cr
View file

@ -29,7 +29,7 @@ struct ChannelVideo
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
page = 1
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = extract_videos(initial_data, author, ucid)
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
views ||= 0_i64
channel_video = videos.select { |video| video.id == video_id }[0]?
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
@ -228,58 +235,56 @@ def fetch_channel(ucid, pull_all_videos : Bool)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
Invidious::Database::Users.add_notification(video)
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end
if pull_all_videos
page += 1
ids = [] of String
loop do
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
videos = extract_videos(initial_data, author, ucid)
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
}) }
videos.each do |video|
ids << video.id
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.live_now,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video)
Invidious::Database::Users.add_notification(video) if was_insert
if was_insert
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
end
end
end
break if count < 25
page += 1
sleep 500.milliseconds
end
end
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
channel.updated = Time.utc
return channel
end

2
src/invidious/channels/community.cr
View file

@ -138,7 +138,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
json.field "title", video_title
json.field "videoId", video_id
json.field "videoThumbnails" do
generate_thumbnails(json, video_id)
Invidious::JSONify::APIv1.thumbnails(json, video_id)
end
json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)

107
src/invidious/channels/playlists.cr
View file

@ -1,93 +1,28 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t }
}
continuation = continuation_items.as_a.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
initial_data = YoutubeAPI.browse(continuation)
else
url = "/channel/#{ucid}/playlists?flow=list&view=1"
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYBCABMAE%3D"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAiABMAE%3D"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
"EglwbGF5bGlzdHMYAyABMAE%3D"
end
case sort_by
when "last", "last_added"
#
when "oldest", "oldest_created"
url += "&sort=da"
when "newest", "newest_created"
url += "&sort=dd"
else nil # Ignore
end
response = YT_POOL.client &.get(url)
initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data
items = extract_items(initial_data, author, ucid)
continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
return items, continuation
end
# ## NOTE: DEPRECATED
# Reason -> Unstable
# The Protobuf object must be provided with an id of the last playlist from the current "page"
# in order to fetch the next one accurately
# (if the id isn't included, entries shift around erratically between pages,
# leading to repetitions and skip overs)
#
# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
# it's better to stick to continuation tokens provided by the first request and onward
def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "playlists",
"6:varint" => 2_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"13:string" => "",
"23:varint" => 0_i64,
},
},
}
if cursor
cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
end
if auto_generated
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
else
object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
case sort
when "oldest", "oldest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
when "newest", "newest_created"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
when "last", "last_added"
object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
else nil # Ignore
end
end
object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
object["80226972:embedded"].delete("3:base64")
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return extract_items(initial_data, author, ucid)
end

166
src/invidious/channels/videos.cr
View file

@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
sort_by_numerical =
case sort_by
when "newest" then 1_i64
when "popular" then 2_i64
when "oldest" then 3_i64 # Broken as of 10/2022 :c
else 1_i64 # Fallback to "newest"
end
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
"1:string" => object_inner_2_encoded,
"2:string" => "00000000-0000-0000-0000-000000000000",
},
"3:varint" => 1_i64,
"3:varint" => sort_by_numerical,
},
},
},
@ -52,34 +60,138 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
return continuation
end
def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
continuation = produce_channel_videos_continuation(ucid, page,
auto_generated: auto_generated, sort_by: sort_by, v2: true)
return YoutubeAPI.browse(continuation)
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo
# 2.times do |i|
# initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by)
videos = extract_videos(initial_data, author, ucid)
# end
return videos.size, videos
end
def get_latest_videos(ucid)
initial_data = get_channel_videos_response(ucid)
author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
return extract_videos(initial_data, author, ucid)
end
# Used in bypass_captcha_job.cr
def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
end
module Invidious::Channel::Tabs
extend self
# -------------------
# Regular videos
# -------------------
def make_initial_video_ctoken(ucid, sort_by) : String
return produce_channel_videos_continuation(ucid, sort_by: sort_by)
end
# Wrapper for AboutChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.ucid,
continuation: continuation, sort_by: sort_by
)
end
# Wrapper for InvidiousChannel, as we still need to call get_videos with
# an author name and ucid directly (e.g in RSS feeds).
# TODO: figure out how to get rid of that
def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
return get_videos(
channel.author, channel.id,
continuation: continuation, sort_by: sort_by
)
end
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_video_ctoken(ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, author, ucid)
end
def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
if continuation.nil?
# Fetch the first "page" of video
items, next_continuation = get_videos(channel, sort_by: sort_by)
else
# Fetch a "page" of videos using the given continuation token
items, next_continuation = get_videos(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_videos(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
# -------------------
# Shorts
# -------------------
private def fetch_shorts_data(ucid : String, continuation : String? = nil)
if continuation.nil?
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
# TODO: try to extract the continuation tokens that allows other sorting options
return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
else
return YoutubeAPI.browse(continuation: continuation)
end
end
def get_shorts(channel : AboutChannel, continuation : String? = nil)
initial_data = self.fetch_shorts_data(channel.ucid, continuation)
begin
# Try to parse the initial data fetched above
return extract_items(initial_data, channel.author, channel.ucid)
rescue ex : RetryOnceException
# Sometimes, for a completely unknown reason, the "reelItemRenderer"
# object is missing some critical information (it happens once in about
# 20 subsequent requests). Refreshing the page is required to properly
# show the "shorts" tab.
#
# In order to make the experience smoother for the user, we simulate
# said page refresh by fetching again the JSON. If that still doesn't
# work, we raise a BrokenTubeException, as something is really broken.
begin
initial_data = self.fetch_shorts_data(channel.ucid, continuation)
return extract_items(initial_data, channel.author, channel.ucid)
rescue ex : RetryOnceException
raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
end
end
end
# -------------------
# Livestreams
# -------------------
def get_livestreams(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation: continuation)
end
return extract_items(initial_data, channel.author, channel.ucid)
end
def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# Fetch the first "page" of streams
items, next_continuation = get_livestreams(channel)
else
# Fetch a "page" of streams using the given continuation token
items, next_continuation = get_livestreams(channel, continuation: continuation)
end
# If there is more to load, then load a second "page"
# and replace the previous continuation token
if !next_continuation.nil?
items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
items.concat items_2
end
return items, next_continuation
end
end

2
src/invidious/config.cr
View file

@ -110,6 +110,8 @@ class Config
property hsts : Bool? = true
# Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
property disable_proxy : Bool? | Array(String)? = false
# Enable the user notifications for all users
property enable_user_notifications : Bool = true
# URL to the modified source code to be easily AGPL compliant
# Will display in the footer, next to the main source code link

10
src/invidious/database/users.cr
View file

@ -154,6 +154,16 @@ module Invidious::Database::Users
# Update (misc)
# -------------------
def feed_needs_update(video : ChannelVideo)
request = <<-SQL
UPDATE users
SET feed_needs_update = true
WHERE $1 = ANY(subscriptions)
SQL
PG_DB.exec(request, video.ucid)
end
def update_preferences(user : User)
request = <<-SQL
UPDATE users

5
src/invidious/exceptions.cr
View file

@ -33,3 +33,8 @@ end
class VideoNotAvailableException < Exception
end
# Exception used to indicate that the JSON response from YT is missing
# some important informations, and that the query should be sent again.
class RetryOnceException < Exception
end

44
src/invidious/frontend/channel_page.cr Normal file
View file

@ -0,0 +1,44 @@
module Invidious::Frontend::ChannelPage
extend self
enum TabsAvailable
Videos
Shorts
Streams
Playlists
Community
Channels
end
def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
return String.build(1500) do |str|
base_url = "/channel/#{channel.ucid}"
TabsAvailable.each do |tab|
# Ignore playlists, as it is not supported for auto-generated channels yet
next if (tab.playlists? && channel.auto_generated)
tab_name = tab.to_s.downcase
if channel.tabs.includes? tab_name
str << %(<div class="pure-u-1 pure-md-1-3">\n)
if tab == selected_tab
str << "\t<b>"
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</b>\n"
else
# Video tab doesn't have the last path component
url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
str << %(\t<a href=") << url << %(">)
str << translate(locale, "channel_tab_#{tab_name}_label")
str << "</a>\n"
end
str << "</div>"
end
end
end
end
end

4
src/invidious/frontend/watch_page.cr
View file

@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
getter captions : Array(Caption)
getter captions : Array(Invidious::Videos::Caption)
def initialize(
@full_videos,
@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage
video_assets.full_videos.each do |option|
mimetype = option["mimeType"].as_s.split(";")[0]
height = itag_to_metadata?(option["itag"]).try &.["height"]?
height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]?
value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json

3
src/invidious/hashtag.cr
View file

@ -8,7 +8,8 @@ module Invidious::Hashtag
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
return extract_items(response)
items, _ = extract_items(response)
return items
end
def generate_continuation(hashtag : String, cursor : Int)

6
src/invidious/helpers/json_filter.cr
View file

@ -20,7 +20,7 @@ module JSONFilter
/^\(|\(\(|\/\(/
end
def self.parse_fields(fields_text : String) : Nil
def self.parse_fields(fields_text : String, &) : Nil
if fields_text.empty?
raise FieldsParser::ParseError.new "Fields is empty"
end
@ -42,7 +42,7 @@ module JSONFilter
parse_nest_groups(fields_text) { |nest_list| yield nest_list }
end
def self.parse_single_nests(fields_text : String) : Nil
def self.parse_single_nests(fields_text : String, &) : Nil
single_nests = remove_nest_groups(fields_text)
if !single_nests.empty?
@ -60,7 +60,7 @@ module JSONFilter
end
end
def self.parse_nest_groups(fields_text : String) : Nil
def self.parse_nest_groups(fields_text : String, &) : Nil
nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
bracket_pairs = get_bracket_pairs(fields_text, true)

11
src/invidious/helpers/serialized_yt_data.cr
View file

@ -76,7 +76,7 @@ struct SearchVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
@ -155,7 +155,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end
end
@ -265,4 +265,11 @@ class Category
end
end
struct Continuation
getter token
def initialize(@token : String)
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

24
src/invidious/helpers/utils.cr
View file

@ -161,21 +161,19 @@ def number_with_separator(number)
number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
end
def short_text_to_number(short_text : String) : Int32
case short_text
when .ends_with? "M"
number = short_text.rstrip(" mM").to_f
number *= 1000000
when .ends_with? "K"
number = short_text.rstrip(" kK").to_f
number *= 1000
else
number = short_text.rstrip(" ")
def short_text_to_number(short_text : String) : Int64
matches = /(?<number>\d+(\.\d+)?)\s?(?<suffix>[mMkKbB])?/.match(short_text)
number = matches.try &.["number"].to_f || 0.0
case matches.try &.["suffix"].downcase
when "k" then number *= 1_000
when "m" then number *= 1_000_000
when "b" then number *= 1_000_000_000
end
number = number.to_i
return number
return number.to_i64
rescue ex
return 0_i64
end
def number_to_short_text(number)

20
src/invidious/http_server/utils.cr Normal file
View file

@ -0,0 +1,20 @@
module Invidious::HttpServer
module Utils
extend self
def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
url = URI.parse(raw_url)
# Add some URL parameters
params = url.query_params
params["host"] = url.host.not_nil! # Should never be nil, in theory
params["region"] = region if !region.nil?
if absolute
return "#{HOST_URL}#{url.request_target}?#{params}"
else
return "#{url.request_target}?#{params}"
end
end
end
end

4
src/invidious/jobs/notification_job.cr
View file

@ -1,12 +1,12 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
connections = [] of Channel(PQ::Notification)
connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }

2
src/invidious/jobs/refresh_channels_job.cr
View file

@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do

2
src/invidious/jobs/refresh_feeds_job.cr
View file

@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|

2
src/invidious/jobs/subscribe_to_feeds_job.cr
View file

@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
active_channel = Channel(Bool).new
active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|

18
src/invidious/jsonify/api_v1/common.cr Normal file
View file

@ -0,0 +1,18 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def thumbnails(json : JSON::Builder, id : String)
json.array do
build_thumbnails(id).each do |thumbnail|
json.object do
json.field "quality", thumbnail[:name]
json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
json.field "width", thumbnail[:width]
json.field "height", thumbnail[:height]
end
end
end
end
end

258
src/invidious/jsonify/api_v1/video_json.cr Normal file
View file

@ -0,0 +1,258 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false)
json.object do
json.field "type", video.video_type
json.field "title", video.title
json.field "videoId", video.id
json.field "error", video.info["reason"] if video.info["reason"]?
json.field "videoThumbnails" do
self.thumbnails(json, video.id)
end
json.field "storyboards" do
self.storyboards(json, video.id, video.storyboards)
end
json.field "description", video.description
json.field "descriptionHtml", video.description_html
json.field "published", video.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
json.field "keywords", video.keywords
json.field "viewCount", video.views
json.field "likeCount", video.likes
json.field "dislikeCount", 0_i64
json.field "paid", video.paid
json.field "premium", video.premium
json.field "isFamilyFriendly", video.is_family_friendly
json.field "allowedRegions", video.allowed_regions
json.field "genre", video.genre
json.field "genreUrl", video.genre_url
json.field "author", video.author
json.field "authorId", video.ucid
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCountText", video.sub_count_text
json.field "lengthSeconds", video.length_seconds
json.field "allowRatings", video.allow_ratings
json.field "rating", 0_i64
json.field "isListed", video.is_listed
json.field "liveNow", video.live_now
json.field "isUpcoming", video.is_upcoming
if video.premiere_timestamp
json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix
end
if hlsvp = video.hls_manifest_url
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}"
json.field "adaptiveFormats" do
json.array do
video.adaptive_fmts.each do |fmt|
json.object do
# Only available on regular videos, not livestreams/OTF streams
if init_range = fmt["initRange"]?
json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
end
if index_range = fmt["indexRange"]?
json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
end
# Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
if proxy
json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
fmt["url"].to_s, absolute: true
)
else
json.field "url", fmt["url"]
end
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]? || "-1"
# Last modified is a unix timestamp with µS, with the dot omitted.
# E.g: 1638056732(.)141582
#
# On livestreams, it's not present, so always fall back to the
# current unix timestamp (up to mS precision) for compatibility.
last_modified = fmt["lastModified"]?
last_modified ||= "#{Time.utc.to_unix_ms.to_s}000"
json.field "lmt", last_modified
json.field "projectionType", fmt["projectionType"]
if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
# Livestream chunk infos
json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
# Audio-related data
json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
# Extra misc stuff
json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
end
end
end
end
json.field "formatStreams" do
json.array do
video.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "captions" do
json.array do
video.captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "language_code", caption.language_code
json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
json.field "recommendedVideos" do
json.array do
video.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
self.thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
json.field "authorUrl", "/channel/#{rv["ucid"]?}"
json.field "authorId", rv["ucid"]?
if rv["author_thumbnail"]?
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
json.field "viewCountText", rv["short_view_count"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
end
end
end
end
def storyboards(json, id, storyboards)
json.array do
storyboards.each do |storyboard|
json.object do
json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
json.field "templateUrl", storyboard[:url]
json.field "width", storyboard[:width]
json.field "height", storyboard[:height]
json.field "count", storyboard[:count]
json.field "interval", storyboard[:interval]
json.field "storyboardWidth", storyboard[:storyboard_width]
json.field "storyboardHeight", storyboard[:storyboard_height]
json.field "storyboardCount", storyboard[:storyboard_count]
end
end
end
end
end

2
src/invidious/playlists.cr
View file

@ -56,7 +56,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index

6
src/invidious/routes/api/manifest.cr
View file

@ -14,8 +14,6 @@ module Invidious::Routes::API::Manifest
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
haltf env, status_code: 404
rescue ex
@ -31,7 +29,7 @@ module Invidious::Routes::API::Manifest
if local
uri = URI.parse(url)
url = "#{uri.request_target}host/#{uri.host}/"
url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
end
"<BaseURL>#{url}</BaseURL>"
@ -44,7 +42,7 @@ module Invidious::Routes::API::Manifest
if local
adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
end

239
src/invidious/routes/api/v1/channels.cr
View file

@ -1,13 +1,7 @@
module Invidious::Routes::API::V1::Channels
def self.home(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
# Macro to avoid duplicating some code below
# This sets the `channel` variable, or handles Exceptions.
private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@ -18,17 +12,25 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
end
page = 1
if channel.auto_generated
videos = [] of SearchVideo
count = 0
else
begin
count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
return error_json(500, ex)
end
def self.home(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
begin
videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
related_channels = fetch_related_channels(channel)
related_channels, _ = fetch_related_channels(channel)
rescue ex
related_channels = [] of AboutRelatedChannel
related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
json.object do
json.field "author", related_channel.author
json.field "authorId", related_channel.ucid
json.field "authorUrl", related_channel.author_url
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
related_channel.to_json(locale, json)
end
end
end # relatedChannels
@ -134,61 +118,112 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
locale = env.get("preferences").as(Preferences).locale
# Remove parameters that could affect this endpoint's behavior
env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
begin
videos = get_latest_videos(ucid)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.array do
videos.each do |video|
video.to_json(locale, json)
end
end
end
return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort"]?.try &.downcase
sort_by ||= env.params.query["sort_by"]?.try &.downcase
sort_by ||= "newest"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve some URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
videos, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
end
begin
count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.array do
videos.each do |video|
video.to_json(locale, json)
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.shorts(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
continuation = env.params.query["continuation"]?
begin
videos, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.streams(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the private macro defined above.
channel = nil # Make the compiler happy
get_channel()
# Retrieve continuation from URL parameters
continuation = env.params.query["continuation"]?
begin
videos, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation
)
rescue ex
return error_json(500, ex)
end
return JSON.build do |json|
json.object do
json.field "videos" do
json.array do
videos.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
end
end
def self.channels(env)
locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
# Use the macro defined above
channel = nil # Make the compiler happy
get_channel()
continuation = env.params.query["continuation"]?
begin
items, next_continuation = fetch_related_channels(channel, continuation)
rescue ex
return error_json(500, ex)
end
JSON.build do |json|
json.object do
json.field "relatedChannels" do
json.array do
items.each &.to_json(locale, json)
end
end
json.field "continuation", next_continuation if next_continuation
end
end
end
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?

2
src/invidious/routes/api/v1/misc.cr
View file

@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Misc
json.field "videoThumbnails" do
json.array do
generate_thumbnails(json, video.id)
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end

16
src/invidious/routes/api/v1/videos.cr
View file

@ -6,19 +6,19 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
video.to_json(locale, nil)
return JSON.build do |json|
Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
end
end
def self.captions(env)
@ -41,9 +41,6 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@ -168,9 +165,6 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@ -185,7 +179,7 @@ module Invidious::Routes::API::V1::Videos
response = JSON.build do |json|
json.object do
json.field "storyboards" do
generate_storyboards(json, id, storyboards)
Invidious::JSONify::APIv1.storyboards(json, id, storyboards)
end
end
end

108
src/invidious/routes/channels.cr
View file

@ -7,21 +7,19 @@ module Invidious::Routes::Channels
def self.videos(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
return data if !data.is_a?(Tuple)
page = env.params.query["page"]?.try &.to_i?
page ||= 1
locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_options = {"last", "oldest", "newest"}
sort_by ||= "last"
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@ -33,34 +31,85 @@ module Invidious::Routes::Channels
items.each(&.author = "")
else
sort_options = {"newest", "oldest", "popular"}
sort_by ||= "newest"
count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_videos(
channel, continuation: continuation, sort_by: (sort_by || "newest")
)
end
selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
templated "channel"
end
def self.shorts(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "shorts"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort option for shorts
sort_by = ""
sort_options = [] of String
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
)
selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
templated "channel"
end
def self.streams(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if !channel.tabs.includes? "streams"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort option for livestreams
sort_by = ""
sort_options = [] of String
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_60_livestreams(
channel, continuation: continuation
)
selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
templated "playlists"
selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
templated "channel"
end
def self.community(env)
@ -74,12 +123,15 @@ module Invidious::Routes::Channels
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
# sort_by = env.params.query["sort_by"]?.try &.downcase
if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
# TODO: support sort options for community posts
sort_by = ""
sort_options = [] of String
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
@ -95,6 +147,26 @@ module Invidious::Routes::Channels
templated "community"
end
def self.channels(env)
data = self.fetch_basic_information(env)
return data if !data.is_a?(Tuple)
locale, user, subscriptions, continuation, ucid, channel = data
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_related_channels(channel, continuation)
# Featured/related channels can't be sorted
sort_options = [] of String
sort_by = nil
selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
templated "channel"
end
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@ -125,7 +197,7 @@ module Invidious::Routes::Channels
end
selected_tab = env.request.path.split("/")[-1]
if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"

4
src/invidious/routes/embed.cr
View file

@ -131,8 +131,6 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
@ -149,7 +147,7 @@ module Invidious::Routes::Embed
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
if notifications && notifications.includes? id
if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)

38
src/invidious/routes/feeds.cr
View file

@ -96,12 +96,14 @@ module Invidious::Routes::Feeds
videos, notifications = get_subscription_feed(user, max_results, page)
# "updated" here is used for delivering new notifications, so if
# we know a user has looked at their feed e.g. in the past 10 minutes,
# they've already seen a video posted 20 minutes ago, and don't need
# to be notified.
Invidious::Database::Users.clear_notifications(user)
user.notifications = [] of String
if CONFIG.enable_user_notifications
# "updated" here is used for delivering new notifications, so if
# we know a user has looked at their feed e.g. in the past 10 minutes,
# they've already seen a video posted 20 minutes ago, and don't need
# to be notified.
Invidious::Database::Users.clear_notifications(user)
user.notifications = [] of String
end
env.set "user", user
templated "feeds/subscriptions"
@ -404,13 +406,15 @@ module Invidious::Routes::Feeds
video = get_video(id, force_refresh: true)
# Deliver notifications to `/api/v1/auth/notifications`
payload = {
"topic" => video.ucid,
"videoId" => video.id,
"published" => published.to_unix,
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
if CONFIG.enable_user_notifications
# Deliver notifications to `/api/v1/auth/notifications`
payload = {
"topic" => video.ucid,
"videoId" => video.id,
"published" => published.to_unix,
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
end
video = ChannelVideo.new({
id: id,
@ -426,7 +430,13 @@ module Invidious::Routes::Feeds
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
Invidious::Database::Users.add_notification(video) if was_insert
if was_insert
if CONFIG.enable_user_notifications
Invidious::Database::Users.add_notification(video)
else
Invidious::Database::Users.feed_needs_update(video)
end
end
end
end

20
src/invidious/routes/video_playback.cr
View file

@ -35,6 +35,13 @@ module Invidious::Routes::VideoPlayback
end
end
# See: https://github.com/iv-org/invidious/issues/3302
range_header = env.request.headers["Range"]?
if range_header.nil?
range_for_head = query_params["range"]? || "0-640"
headers["Range"] = "bytes=#{range_for_head}"
end
client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
@ -70,6 +77,9 @@ module Invidious::Routes::VideoPlayback
end
end
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
@ -91,14 +101,8 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}"
if region
location += "&region=#{region}"
end
return env.redirect location
url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
return env.redirect url
end
IO.copy(resp.body_io, env.response)

4
src/invidious/routes/watch.cr
View file

@ -61,8 +61,6 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)
@ -82,7 +80,7 @@ module Invidious::Routes::Watch
Invidious::Database::Users.mark_watched(user.as(User), id)
end
if notifications && notifications.includes? id
if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)

19
src/invidious/routing.cr
View file

@ -37,7 +37,9 @@ module Invidious::Routing
get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get
post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post
get "/modify_notifications", Routes::Notifications, :modify
if CONFIG.enable_user_notifications
get "/modify_notifications", Routes::Notifications, :modify
end
{% end %}
self.register_image_routes
@ -115,14 +117,17 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
get "/channel/:ucid/shorts", Routes::Channels, :shorts
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
["", "/videos", "/playlists", "/community", "/about"].each do |path|
{"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
# /c/LinusTechTips
get "/c/:user#{path}", Routes::Channels, :brand_redirect
# /user/linustechtips | Not always the same as /c/
@ -220,6 +225,10 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
{% for route in {"videos", "latest", "playlists", "community", "search"} %}
get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
@ -260,8 +269,10 @@ module Invidious::Routing
post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
if CONFIG.enable_user_notifications
get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
end
# Misc
get "/api/v1/stats", {{namespace}}::Misc, :stats

14
src/invidious/search/processors.cr
View file

@ -9,7 +9,8 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
return extract_items(initial_data)
items, _ = extract_items(initial_data)
return items
end
# Search a youtube channel
@ -30,16 +31,7 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
items, _ = extract_items(response_json, "", ucid)
return items
end

1141
src/invidious/videos.cr
View file

File diff suppressed because it is too large Load diff

168
src/invidious/videos/caption.cr Normal file
View file

@ -0,0 +1,168 @@
require "json"
module Invidious::Videos
struct Caption
property name : String
property language_code : String
property base_url : String
def initialize(@name, @language_code, @base_url)
end
# Parse the JSON structure from Youtube
def self.from_yt_json(container : JSON::Any) : Array(Caption)
caption_tracks = container
.dig?("playerCaptionsTracklistRenderer", "captionTracks")
.try &.as_a
captions_list = [] of Caption
return captions_list if caption_tracks.nil?
caption_tracks.each do |caption|
name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
name = name.to_s.split(" - ")[0]
language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s
captions_list << Caption.new(name, language_code, base_url)
end
return captions_list
end
# List of all caption languages available on Youtube.
LANGUAGES = {
"",
"English",
"English (auto-generated)",
"English (United Kingdom)",
"English (United States)",
"Afrikaans",
"Albanian",
"Amharic",
"Arabic",
"Armenian",
"Azerbaijani",
"Bangla",
"Basque",
"Belarusian",
"Bosnian",
"Bulgarian",
"Burmese",
"Cantonese (Hong Kong)",
"Catalan",
"Cebuano",
"Chinese",
"Chinese (China)",
"Chinese (Hong Kong)",
"Chinese (Simplified)",
"Chinese (Taiwan)",
"Chinese (Traditional)",
"Corsican",
"Croatian",
"Czech",
"Danish",
"Dutch",
"Dutch (auto-generated)",
"Esperanto",
"Estonian",
"Filipino",
"Finnish",
"French",
"French (auto-generated)",
"Galician",
"Georgian",
"German",
"German (auto-generated)",
"Greek",
"Gujarati",
"Haitian Creole",
"Hausa",
"Hawaiian",
"Hebrew",
"Hindi",
"Hmong",
"Hungarian",
"Icelandic",
"Igbo",
"Indonesian",
"Indonesian (auto-generated)",
"Interlingue",
"Irish",
"Italian",
"Italian (auto-generated)",
"Japanese",
"Japanese (auto-generated)",
"Javanese",
"Kannada",
"Kazakh",
"Khmer",
"Korean",
"Korean (auto-generated)",
"Kurdish",
"Kyrgyz",
"Lao",
"Latin",
"Latvian",
"Lithuanian",
"Luxembourgish",
"Macedonian",
"Malagasy",
"Malay",
"Malayalam",
"Maltese",
"Maori",
"Marathi",
"Mongolian",
"Nepali",
"Norwegian Bokmål",
"Nyanja",
"Pashto",
"Persian",
"Polish",
"Portuguese",
"Portuguese (auto-generated)",
"Portuguese (Brazil)",
"Punjabi",
"Romanian",
"Russian",
"Russian (auto-generated)",
"Samoan",
"Scottish Gaelic",
"Serbian",
"Shona",
"Sindhi",
"Sinhala",
"Slovak",
"Slovenian",
"Somali",
"Southern Sotho",
"Spanish",
"Spanish (auto-generated)",
"Spanish (Latin America)",
"Spanish (Mexico)",
"Spanish (Spain)",
"Sundanese",
"Swahili",
"Swedish",
"Tajik",
"Tamil",
"Telugu",
"Thai",
"Turkish",
"Turkish (auto-generated)",
"Ukrainian",
"Urdu",
"Uzbek",
"Vietnamese",
"Vietnamese (auto-generated)",
"Welsh",
"Western Frisian",
"Xhosa",
"Yiddish",
"Yoruba",
"Zulu",
}
end
end

116
src/invidious/videos/formats.cr Normal file
View file

@ -0,0 +1,116 @@
module Invidious::Videos::Formats
def self.itag_to_metadata?(itag : JSON::Any)
return FORMATS[itag.to_s]?
end
# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
private FORMATS = {
"5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
"17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
"18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
"22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
"37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
# 3D videos
"82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
# Apple HTTP Live Streaming
"91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
# DASH mp4 video
"133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
"134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
"135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
"137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
"138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
"160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
"212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
"298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
# Dash mp4 audio
"139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
"140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
"141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
"256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
"328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
# Dash webm
"167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
"242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
"243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
"244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
"248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
"271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
# itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
"272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
# Dash webm audio
"171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
"172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
# Dash webm audio with opus inside
"249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
"250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
"251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
# av01 video only formats sometimes served with "unknown" codecs
"394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
"395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
"396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
"397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
}
end

373
src/invidious/videos/parser.cr Normal file
View file

@ -0,0 +1,373 @@
require "json"
# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
# The former is preferred as it has more videos in it. The second has
# the same 11 first entries as the compact rendered.
#
# TODO: "compactRadioRenderer" (Mix) and
# TODO: Use a proper struct/class instead of a hacky JSON object
def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
return nil if !related["videoId"]?
# The compact renderer has video length in seconds, where the end
# screen rendered has a full text version ("42:40")
length = related["lengthInSeconds"]?.try &.as_i.to_s
length ||= related.dig?("lengthText", "simpleText").try do |box|
decode_length_seconds(box.as_s).to_s
end
# Both have "short", so the "long" option shouldn't be required
channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
.try &.dig?("runs", 0)
author = channel_info.try &.dig?("text")
author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
# "4,088,033 views", only available on compact renderer
# and when video is not a livestream
view_count = related.dig?("viewCountText", "simpleText")
.try &.as_s.gsub(/\D/, "")
short_view_count = related.try do |r|
HelperExtractors.get_short_view_count(r).to_s
end
LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
# TODO: when refactoring video types, make a struct for related videos
# or reuse an existing type, if that fits.
return {
"id" => related["videoId"],
"title" => related["title"]["simpleText"],
"author" => author || JSON::Any.new(""),
"ucid" => JSON::Any.new(ucid || ""),
"length_seconds" => JSON::Any.new(length || "0"),
"view_count" => JSON::Any.new(view_count || "0"),
"short_view_count" => JSON::Any.new(short_view_count || "0"),
"author_verified" => JSON::Any.new(author_verified),
}
end
def extract_video_info(video_id : String, proxy_region : String? = nil)
# Init client config for the API
client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
# Fetch data from the player endpoint
# 8AEB param is used to fetch YouTube stories
player_response = YoutubeAPI.player(video_id: video_id, params: "8AEB", client_config: client_config)
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
if playability_status != "OK"
subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
reason = subreason.try &.[]?("simpleText").try &.as_s
reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
reason ||= player_response.dig("playabilityStatus", "reason").as_s
# Stop here if video is not a scheduled livestream or
# for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new(reason),
}
end
elsif video_id != player_response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)")
else
reason = nil
end
# Don't fetch the next endpoint if the video is unavailable.
if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
player_response = player_response.merge(next_response)
end
params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason
new_player_response = nil
if reason.nil?
# Fetch the video streams using an Android client in order to get the
# decrypted URLs and maybe fix throttling issues (#2194). See the
# following issue for an explanation about decrypted URLs:
# https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
client_config.client_type = YoutubeAPI::ClientType::Android
new_player_response = try_fetch_streaming_data(video_id, client_config)
elsif !reason.includes?("your country") # Handled separately
# The Android embedded client could help here
client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Last hope
if new_player_response.nil?
client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Replace player response and reset reason
if !new_player_response.nil?
player_response = new_player_response
params.delete("reason")
end
{"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
params[f] = player_response[f] if player_response[f]?
end
# Data structure version, for cache control
params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
return params
end
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
# 8AEB param is used to fetch YouTube stories
response = YoutubeAPI.player(video_id: id, params: "8AEB", client_config: client_config)
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
if id != response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise VideoNotAvailableException.new(
"The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
)
elsif playability_status == "OK"
return response
else
return nil
end
end
def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
# Top level elements
main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
# Primary results are not available on Music videos
# See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
if primary_results = main_results.dig?("results", "results", "contents")
video_primary_renderer = primary_results
.as_a.find(&.["videoPrimaryInfoRenderer"]?)
.try &.["videoPrimaryInfoRenderer"]
video_secondary_renderer = primary_results
.as_a.find(&.["videoSecondaryInfoRenderer"]?)
.try &.["videoSecondaryInfoRenderer"]
raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
end
video_details = player_response.dig?("videoDetails")
microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
raise BrokenTubeException.new("videoDetails") if !video_details
raise BrokenTubeException.new("microformat") if !microformat
# Basic video infos
title = video_details["title"]?.try &.as_s
# We have to try to extract viewCount from videoPrimaryInfoRenderer first,
# then from videoDetails, as the latter is "0" for livestreams (we want
# to get the amount of viewers watching).
views_txt = video_primary_renderer
.try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text")
views_txt ||= video_details["viewCount"]?
views = views_txt.try &.as_s.gsub(/\D/, "").to_i64?
length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
.try &.as_s.to_i64
published = microformat["publishDate"]?
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
.try &.as_bool || false
# Extra video infos
allowed_regions = microformat["availableCountries"]?
.try &.as_a.map &.as_s || [] of String
allow_ratings = video_details["allowRatings"]?.try &.as_bool
family_friendly = microformat["isFamilySafe"].try &.as_bool
is_listed = video_details["isCrawlable"]?.try &.as_bool
is_upcoming = video_details["isUpcoming"]?.try &.as_bool
keywords = video_details["keywords"]?
.try &.as_a.map &.as_s || [] of String
# Related videos
LOGGER.debug("extract_video_info: parsing related videos...")
related = [] of JSON::Any
# Parse "compactVideoRenderer" items (under secondary results)
secondary_results = main_results
.dig?("secondaryResults", "secondaryResults", "results")
secondary_results.try &.as_a.each do |element|
if item = element["compactVideoRenderer"]?
related_video = parse_related_video(item)
related << JSON::Any.new(related_video) if related_video
end
end
# If nothing was found previously, fall back to end screen renderer
if related.empty?
# Container for "endScreenVideoRenderer" items
player_overlays = player_response.dig?(
"playerOverlays", "playerOverlayRenderer",
"endScreen", "watchNextEndScreenRenderer", "results"
)
player_overlays.try &.as_a.each do |element|
if item = element["endScreenVideoRenderer"]?
related_video = parse_related_video(item)
related << JSON::Any.new(related_video) if related_video
end
end
end
# Likes
toplevel_buttons = video_primary_renderer
.try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
if toplevel_buttons
likes_button = toplevel_buttons.try &.as_a
.find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
.try &.["toggleButtonRenderer"]
# New format as of september 2022
likes_button ||= toplevel_buttons.try &.as_a
.find(&.["segmentedLikeDislikeButtonRenderer"]?)
.try &.dig?(
"segmentedLikeDislikeButtonRenderer",
"likeButton", "toggleButtonRenderer"
)
if likes_button
# Note: The like count from `toggledText` is off by one, as it would
# represent the new like count in the event where the user clicks on "like".
likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
.try &.dig?("accessibility", "accessibilityData", "label")
likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
end
end
# Description
description = microformat.dig?("description", "simpleText").try &.as_s || ""
short_description = player_response.dig?("videoDetails", "shortDescription")
description_html = video_secondary_renderer.try &.dig?("description", "runs")
.try &.as_a.try { |t| content_to_comment_html(t, video_id) }
# Video metadata
metadata = video_secondary_renderer
.try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
.try &.as_a
genre = microformat["category"]?
genre_ucid = nil
license = nil
metadata.try &.each do |row|
metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
contents = row.dig?("metadataRowRenderer", "contents", 0)
if metadata_title == "Category"
contents = contents.try &.dig?("runs", 0)
genre = contents.try &.["text"]?
genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
elsif metadata_title == "License"
license = contents.try &.dig?("runs", 0, "text")
elsif metadata_title == "Licensed to YouTube by"
license = contents.try &.["simpleText"]?
end
end
# Author infos
author = video_details["author"]?.try &.as_s
ucid = video_details["channelId"]?.try &.as_s
if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
author_verified = has_verified_badge?(author_info["badges"]?)
subs_text = author_info["subscriberCountText"]?
.try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
.try &.as_s.split(" ", 2)[0]
end
# Return data
if live_now
video_type = VideoType::Livestream
elsif !premiere_timestamp.nil?
video_type = VideoType::Scheduled
published = premiere_timestamp || Time.utc
else
video_type = VideoType::Video
end
params = {
"videoType" => JSON::Any.new(video_type.to_s),
# Basic video infos
"title" => JSON::Any.new(title || ""),
"views" => JSON::Any.new(views || 0_i64),
"likes" => JSON::Any.new(likes || 0_i64),
"lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
"published" => JSON::Any.new(published.to_rfc3339),
# Extra video infos
"allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
"allowRatings" => JSON::Any.new(allow_ratings || false),
"isFamilyFriendly" => JSON::Any.new(family_friendly || false),
"isListed" => JSON::Any.new(is_listed || false),
"isUpcoming" => JSON::Any.new(is_upcoming || false),
"keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
# Related videos
"relatedVideos" => JSON::Any.new(related),
# Description
"description" => JSON::Any.new(description || ""),
"descriptionHtml" => JSON::Any.new(description_html || "<p></p>"),
"shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
# Video metadata
"genre" => JSON::Any.new(genre.try &.as_s || ""),
"genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
"license" => JSON::Any.new(license.try &.as_s || ""),
# Author infos
"author" => JSON::Any.new(author || ""),
"ucid" => JSON::Any.new(ucid || ""),
"authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
"authorVerified" => JSON::Any.new(author_verified || false),
"subCountText" => JSON::Any.new(subs_text || "-"),
}
return params
end

27
src/invidious/videos/regions.cr Normal file
View file

@ -0,0 +1,27 @@
# List of geographical regions that Youtube recognizes.
# This is used to determine if a video is either restricted to a list
# of allowed regions (= whitelisted) or if it can't be watched in
# a set of regions (= blacklisted).
REGIONS = {
"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
"AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
"BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
"BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
"CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
"DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
"FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
"GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
"HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
"IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
"KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
"LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
"ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
"MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
"NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
"PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
"SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
"SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
"TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
"TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
"VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
}

156
src/invidious/videos/video_preferences.cr Normal file
View file

@ -0,0 +1,156 @@
struct VideoPreferences
include JSON::Serializable
property annotations : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
property continue_autoplay : Bool
property controls : Bool
property listen : Bool
property local : Bool
property preferred_captions : Array(String)
property player_style : String
property quality : String
property quality_dash : String
property raw : Bool
property region : String?
property related_videos : Bool
property speed : Float32 | Float64
property video_end : Float64 | Int32
property video_loop : Bool
property extend_desc : Bool
property video_start : Float64 | Int32
property volume : Int32
property vr_mode : Bool
property save_player_pos : Bool
end
def process_video_params(query, preferences)
annotations = query["iv_load_policy"]?.try &.to_i?
autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
comments = query["comments"]?.try &.split(",").map(&.downcase)
continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
player_style = query["player_style"]?
preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
quality = query["quality"]?
quality_dash = query["quality_dash"]?
region = query["region"]?
related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
speed = query["speed"]?.try &.rchop("x").to_f?
video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
volume = query["volume"]?.try &.to_i?
vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
if preferences
# region ||= preferences.region
annotations ||= preferences.annotations.to_unsafe
autoplay ||= preferences.autoplay.to_unsafe
comments ||= preferences.comments
continue ||= preferences.continue.to_unsafe
continue_autoplay ||= preferences.continue_autoplay.to_unsafe
listen ||= preferences.listen.to_unsafe
local ||= preferences.local.to_unsafe
player_style ||= preferences.player_style
preferred_captions ||= preferences.captions
quality ||= preferences.quality
quality_dash ||= preferences.quality_dash
related_videos ||= preferences.related_videos.to_unsafe
speed ||= preferences.speed
video_loop ||= preferences.video_loop.to_unsafe
extend_desc ||= preferences.extend_desc.to_unsafe
volume ||= preferences.volume
vr_mode ||= preferences.vr_mode.to_unsafe
save_player_pos ||= preferences.save_player_pos.to_unsafe
end
annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
comments ||= CONFIG.default_user_preferences.comments
continue ||= CONFIG.default_user_preferences.continue.to_unsafe
continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
listen ||= CONFIG.default_user_preferences.listen.to_unsafe
local ||= CONFIG.default_user_preferences.local.to_unsafe
player_style ||= CONFIG.default_user_preferences.player_style
preferred_captions ||= CONFIG.default_user_preferences.captions
quality ||= CONFIG.default_user_preferences.quality
quality_dash ||= CONFIG.default_user_preferences.quality_dash
related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
speed ||= CONFIG.default_user_preferences.speed
video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
volume ||= CONFIG.default_user_preferences.volume
vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
annotations = annotations == 1
autoplay = autoplay == 1
continue = continue == 1
continue_autoplay = continue_autoplay == 1
listen = listen == 1
local = local == 1
related_videos = related_videos == 1
video_loop = video_loop == 1
extend_desc = extend_desc == 1
vr_mode = vr_mode == 1
save_player_pos = save_player_pos == 1
if CONFIG.disabled?("dash") && quality == "dash"
quality = "high"
end
if CONFIG.disabled?("local") && local
local = false
end
if start = query["t"]? || query["time_continue"]? || query["start"]?
video_start = decode_time(start)
end
video_start ||= 0
if query["end"]?
video_end = decode_time(query["end"])
end
video_end ||= -1
raw = query["raw"]?.try &.to_i?
raw ||= 0
raw = raw == 1
controls = query["controls"]?.try &.to_i?
controls ||= 1
controls = controls >= 1
params = VideoPreferences.new({
annotations: annotations,
autoplay: autoplay,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
controls: controls,
listen: listen,
local: local,
player_style: player_style,
preferred_captions: preferred_captions,
quality: quality,
quality_dash: quality_dash,
raw: raw,
region: region,
related_videos: related_videos,
speed: speed,
video_end: video_end,
video_loop: video_loop,
extend_desc: extend_desc,
video_start: video_start,
volume: volume,
vr_mode: vr_mode,
save_player_pos: save_player_pos,
})
return params
end

122
src/invidious/views/channel.ecr
View file

@ -1,8 +1,24 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
<%-
ucid = channel.ucid
author = HTML.escape(channel.author)
channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
relative_url =
case selected_tab
when .shorts? then "/channel/#{ucid}/shorts"
when .streams? then "/channel/#{ucid}/streams"
when .playlists? then "/channel/#{ucid}/playlists"
when .channels? then "/channel/#{ucid}/channels"
else
"/channel/#{ucid}"
end
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-%>
<% content_for "header" do %>
<%- if selected_tab.videos? -%>
<meta name="description" content="<%= channel.description %>">
<meta property="og:site_name" content="Invidious">
<meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
@ -14,91 +30,14 @@
<meta name="twitter:title" content="<%= author %>">
<meta name="twitter:description" content="<%= channel.description %>">
<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>">
<title><%= author %> - Invidious</title>
<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
<%- end -%>
<link rel="alternate" href="<%= youtube_url %>">
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= channel_profile_pic %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-3">
<a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<% if !channel.auto_generated %>
<div class="pure-u-1 pure-md-1-3">
<b><%= translate(locale, "Videos") %></b>
</div>
<% end %>
<div class="pure-u-1 pure-md-1-3">
<% if channel.auto_generated %>
<b><%= translate(locale, "Playlists") %></b>
<% else %>
<a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
<% end %>
</div>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
<div class="pure-g" style="text-align:right">
<% sort_options.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>
<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>
@ -113,17 +52,10 @@
<script src="/js/watched_widget.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %>
<a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<%= translate(locale, "Previous page") %>
</a>
<% end %>
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count == 60 %>
<a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<% if next_continuation %>
<a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>

76
src/invidious/views/community.ecr
View file

@ -1,71 +1,21 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<%-
ucid = channel.ucid
author = HTML.escape(channel.author)
channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
relative_url = "/channel/#{ucid}/community"
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
-%>
<% content_for "header" do %>
<link rel="alternate" href="<%= youtube_url %>">
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3 style="text-align:right">
<a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-3">
<a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<% if !channel.auto_generated %>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a>
</div>
<% end %>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<b><%= translate(locale, "Community") %></b>
<% end %>
</div>
</div>
<div class="pure-u-2-3"></div>
</div>
<%= rendered "components/channel_info" %>
<div class="h-box">
<hr>

60
src/invidious/views/components/channel_info.ecr Normal file
View file

@ -0,0 +1,60 @@
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= channel_profile_pic %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-u-1-2">
<div class="pure-u-1 pure-md-1-3">
<a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a>
</div>
<%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
</div>
<div class="pure-u-1-2">
<div class="pure-g" style="text-align:end">
<% sort_options.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>

4
src/invidious/views/feeds/subscriptions.ecr
View file

@ -23,6 +23,8 @@
</div>
</div>
<% if CONFIG.enable_user_notifications %>
<center>
<%= translate_count(locale, "subscriptions_unseen_notifs_count", notifications.size) %>
</center>
@ -39,6 +41,8 @@
<% end %>
</div>
<% end %>
<div class="h-box">
<hr>
</div>

110
src/invidious/views/playlists.ecr
View file

@ -1,110 +0,0 @@
<% ucid = channel.ucid %>
<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
<title><%= author %> - Invidious</title>
<% end %>
<% if channel.banner %>
<div class="h-box">
<img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
</div>
<div class="h-box">
<hr>
</div>
<% end %>
<div class="pure-g h-box">
<div class="pure-u-2-3">
<div class="channel-profile">
<img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
<span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
</div>
</div>
<div class="pure-u-1-3" style="text-align:right">
<h3 style="text-align:right">
<a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
</h3>
</div>
</div>
<div class="h-box">
<div id="descriptionWrapper">
<p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p>
</div>
</div>
<div class="h-box">
<% sub_count_text = number_to_short_text(channel.sub_count) %>
<%= rendered "components/subscribe_widget" %>
</div>
<div class="pure-g h-box">
<div class="pure-g pure-u-1-3">
<div class="pure-u-1 pure-md-1-3">
<a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% else %>
<a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if !channel.auto_generated %>
<b><%= translate(locale, "Playlists") %></b>
<% end %>
</div>
<div class="pure-u-1 pure-md-1-3">
<% if channel.tabs.includes? "community" %>
<a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
<% end %>
</div>
</div>
<div class="pure-u-1-3"></div>
<div class="pure-u-1-3">
<div class="pure-g" style="text-align:right">
<% {"last", "oldest", "newest"}.each do |sort| %>
<div class="pure-u-1 pure-md-1-3">
<% if sort_by == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</div>
<div class="h-box">
<hr>
</div>
<div class="pure-g">
<% items.each do |item| %>
<%= rendered "components/item" %>
<% end %>
</div>
<script src="/js/watched_widget.js"></script>
<div class="pure-g h-box">
<div class="pure-u-1 pure-u-md-4-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if continuation %>
<a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
<%= translate(locale, "Next page") %>
</a>
<% end %>
</div>
</div>

4
src/invidious/views/template.ecr
View file

@ -54,7 +54,7 @@
<div class="pure-u-1-4">
<a id="notification_ticker" title="<%= translate(locale, "Subscriptions") %>" href="/feed/subscriptions" class="pure-menu-heading">
<% notification_count = env.get("user").as(Invidious::User).notifications.size %>
<% if notification_count > 0 %>
<% if CONFIG.enable_user_notifications && notification_count > 0 %>
<span id="notification_count"><%= notification_count %></span> <i class="icon ion-ios-notifications"></i>
<% else %>
<i class="icon ion-ios-notifications-outline"></i>
@ -170,7 +170,9 @@
}.to_pretty_json
%>
</script>
<% if CONFIG.enable_user_notifications %>
<script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script>
<% end %>
<% end %>
</body>

2
src/invidious/views/user/data_control.ecr
View file

@ -14,7 +14,7 @@
<div class="pure-control-group">
<label for="import_youtube">
<a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/Export-YouTube-subscriptions.md">
<a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/docs/export-youtube-subscriptions.md">
<%= translate(locale, "Import YouTube subscriptions") %>
</a>
</label>

4
src/invidious/views/user/preferences.ecr
View file

@ -89,7 +89,7 @@
<label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
<% preferences.captions.each_with_index do |caption, index| %>
<select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
<% CAPTION_LANGUAGES.each do |option| %>
<% Invidious::Videos::Caption::LANGUAGES.each do |option| %>
<option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
<% end %>
</select>
@ -244,6 +244,7 @@
<input name="unseen_only" id="unseen_only" type="checkbox" <% if preferences.unseen_only %>checked<% end %>>
</div>
<% if CONFIG.enable_user_notifications %>
<div class="pure-control-group">
<label for="notifications_only"><%= translate(locale, "preferences_notifications_only_label") %></label>
<input name="notifications_only" id="notifications_only" type="checkbox" <% if preferences.notifications_only %>checked<% end %>>
@ -255,6 +256,7 @@
<a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a>
</div>
<% end %>
<% end %>
<% end %>
<% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(Invidious::User).email %>

134
src/invidious/yt_backend/extractors.cr
View file

@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
private ITEM_CONTAINER_EXTRACTOR = {
Extractors::YouTubeTabs,
Extractors::SearchResults,
Extractors::Continuation,
Extractors::ContinuationContent,
}
private ITEM_PARSERS = {
@ -18,8 +18,11 @@ private ITEM_PARSERS = {
Parsers::CategoryRendererParser,
Parsers::RichItemRendererParser,
Parsers::ReelItemRendererParser,
Parsers::ContinuationItemRendererParser,
}
private alias InitialData = Hash(String, JSON::Any)
record AuthorFallback, name : String, id : String
# Namespace for logic relating to parsing InnerTube data into various datastructs.
@ -170,7 +173,7 @@ private module Parsers
# Always simpleText
# TODO change default value to nil
subscriber_count = item_contents.dig?("subscriberCountText", "simpleText")
.try { |s| short_text_to_number(s.as_s.split(" ")[0]) } || 0
.try { |s| short_text_to_number(s.as_s.split(" ")[0]).to_i32 } || 0
# Auto-generated channels doesn't have videoCountText
# Taken from: https://github.com/iv-org/invidious/pull/2228#discussion_r717620922
@ -345,14 +348,9 @@ private module Parsers
content_container = item_contents["contents"]
end
raw_contents = content_container["items"]?.try &.as_a
if !raw_contents.nil?
raw_contents.each do |item|
result = extract_item(item)
if !result.nil?
contents << result
end
end
content_container["items"]?.try &.as_a.each do |item|
result = parse_item(item, author_fallback.name, author_fallback.id)
contents << result if result.is_a?(SearchItem)
end
Category.new({
@ -384,7 +382,9 @@ private module Parsers
end
private def self.parse(item_contents, author_fallback)
return VideoRendererParser.process(item_contents, author_fallback)
child = VideoRendererParser.process(item_contents, author_fallback)
child ||= ReelItemRendererParser.process(item_contents, author_fallback)
return child
end
def self.parser_name
@ -408,9 +408,19 @@ private module Parsers
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
video_details_container = item_contents.dig(
reel_player_overlay = item_contents.dig(
"navigationEndpoint", "reelWatchEndpoint",
"overlay", "reelPlayerOverlayRenderer",
"overlay", "reelPlayerOverlayRenderer"
)
# Sometimes, the "reelPlayerOverlayRenderer" object is missing the
# important part of the response. We use this exception to tell
# the calling function to fetch the content again.
if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
raise RetryOnceException.new
end
video_details_container = reel_player_overlay.dig(
"reelPlayerHeaderSupportedRenderers",
"reelPlayerHeaderRenderer"
)
@ -436,9 +446,9 @@ private module Parsers
# View count
view_count_text = video_details_container.dig?("viewCountText", "simpleText")
view_count_text ||= video_details_container
.dig?("viewCountText", "accessibility", "accessibilityData", "label")
# View count used to be in the reelWatchEndpoint, but that changed?
view_count_text = item_contents.dig?("viewCountText", "simpleText")
view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
@ -450,8 +460,8 @@ private module Parsers
regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data)
minutes = regex_match.try &.["min"].to_i(strict: false) || 0
seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
duration = (minutes*60 + seconds)
@ -475,6 +485,35 @@ private module Parsers
return {{@type.name}}
end
end
# Parses an InnerTube continuationItemRenderer into a Continuation.
# Returns nil when the given object isn't a continuationItemRenderer.
#
# continuationItemRenderer contains various metadata ued to load more
# content (i.e when the user scrolls down). The interesting bit is the
# protobuf object known as the "continutation token". Previously, those
# were generated from sratch, but recent (as of 11/2022) Youtube changes
# are forcing us to extract them from replies.
#
module ContinuationItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["continuationItemRenderer"]?
return self.parse(item_contents)
end
end
private def self.parse(item_contents)
token = item_contents
.dig?("continuationEndpoint", "continuationCommand", "token")
.try &.as_s
return Continuation.new(token) if token
end
def self.parser_name
return {{@type.name}}
end
end
end
# The following are the extractors for extracting an array of items from
@ -510,7 +549,7 @@ private module Extractors
# }]
#
module YouTubeTabs
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
@ -575,7 +614,7 @@ private module Extractors
# }
#
module SearchResults
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
@ -608,8 +647,8 @@ private module Extractors
# The way they are structured is too varied to be accurately written down here.
# However, they all eventually lead to an array of parsable items after traversing
# through the JSON structure.
module Continuation
def self.process(initial_data : Hash(String, JSON::Any))
module ContinuationContent
def self.process(initial_data : InitialData)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@ -691,8 +730,7 @@ end
# Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
def extract_item(item : JSON::Any, author_fallback : String? = "",
author_id_fallback : String? = "")
def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
# We "allow" nil values but secretly use empty strings instead. This is to save us the
# hassle of modifying every author_fallback and author_id_fallback arg usage
# which is more often than not nil.
@ -702,24 +740,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
# Each parser automatically validates the data given to see if the data is
# applicable to itself. If not nil is returned and the next parser is attempted.
ITEM_PARSERS.each do |parser|
LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
if result = parser.process(item, author_fallback)
LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
return result
else
LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
end
end
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) : Array(SearchItem)
items = [] of SearchItem
#
# This function yields the container so that items can be parsed separately.
#
def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@ -727,24 +764,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
unpackaged_data = initial_data
end
# This is identical to the parser cycling of extract_item().
# This is identical to the parser cycling of parse_item().
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
if container = extractor.process(unpackaged_data)
LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
# Extract items in container
container.each do |item|
if parsed_result = extract_item(item, author_fallback, author_id_fallback)
items << parsed_result
end
end
break
container.each { |item| yield item }
else
LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
end
end
return items
end
# Wrapper using the block function above
def extract_items(
initial_data : InitialData,
author_fallback : String? = nil,
author_id_fallback : String? = nil
) : {Array(SearchItem), String?}
items = [] of SearchItem
continuation = nil
extract_items(initial_data) do |item|
parsed = parse_item(item, author_fallback, author_id_fallback)
case parsed
when .is_a?(Continuation) then continuation = parsed.token
when .is_a?(SearchItem) then items << parsed
end
end
return items, continuation
end

27
src/invidious/yt_backend/extractors_utils.cr
View file

@ -68,10 +68,10 @@ rescue ex
return false
end
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
target = [] of SearchItem
target = [] of (SearchItem | Continuation)
extracted.each do |i|
if i.is_a?(Category)
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
target << i
end
end
return target.select(SearchVideo).map(&.as(SearchVideo))
return target.select(SearchVideo)
end
def extract_selected_tab(tabs)
# Extract the selected tab from the array of tabs Youtube returns
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
end
def fetch_continuation_token(items : Array(JSON::Any))
# Fetches the continuation token from an array of items
return items.last["continuationItemRenderer"]?
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
end
def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
# Fetches the continuation token from initial data
if initial_data["onResponseReceivedActions"]?
continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
else
tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
end
return fetch_continuation_token(continuation_items.as_a)
end

2
src/invidious/yt_backend/youtube_api.cr
View file

@ -43,7 +43,7 @@ module YoutubeAPI
ClientType::Web => {
name: "WEB",
name_proto: "1",
version: "2.20220804.07.00",
version: "2.20221118.01.00",
api_key: DEFAULT_API_KEY,
screen: "WATCH_FULL_SCREEN",
os_name: "Windows",