Migrate more types to data_structs

This commit is contained in:
syeopite 2021-08-08 16:34:30 -07:00
parent 2333221e14
commit df1e4888cd
No known key found for this signature in database
GPG key ID: 6FA616E5A5294A82
30 changed files with 142 additions and 1286 deletions

View file

@ -113,17 +113,17 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
if CONFIG.check_tables
check_enum(PG_DB, "privacy", PlaylistPrivacy)
check_table(PG_DB, "channels", InvidiousChannel)
check_table(PG_DB, "channel_videos", ChannelVideo)
check_table(PG_DB, "playlists", InvidiousPlaylist)
check_table(PG_DB, "playlist_videos", PlaylistVideo)
check_table(PG_DB, "channels", InvidiousStructs::Channel)
check_table(PG_DB, "channel_videos", InvidiousStructs::ChannelVideo)
check_table(PG_DB, "playlists", InvidiousStructs::Playlist)
check_table(PG_DB, "playlist_videos", YouTubeStructs::PlaylistVideo)
check_table(PG_DB, "nonces", Nonce)
check_table(PG_DB, "session_ids", SessionId)
check_table(PG_DB, "users", User)
check_table(PG_DB, "videos", Video)
check_table(PG_DB, "videos", YouTubeStructs::Video)
if CONFIG.cache_annotations
check_table(PG_DB, "annotations", Annotation)
check_table(PG_DB, "annotations", YouTubeStructs::Annotation)
end
end
@ -646,14 +646,14 @@ get "/subscription_manager" do |env|
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel)
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel)
subscriptions.sort_by! { |channel| channel.author.downcase }
if action_takeout
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
next JSON.build do |json|
json.object do
@ -795,7 +795,7 @@ post "/data_control" do |env|
next
end
playlist_video = PlaylistVideo.new({
playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,

View file

@ -1,32 +1,3 @@
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
include DB::Serializable
property ucid : String
property author : String
property auto_generated : Bool
property author_url : String
property author_thumbnail : String
property banner : String?
property description_html : String
property total_views : Int64
property sub_count : Int32
property joined : Time
property is_family_friendly : Bool
property allowed_regions : Array(String)
property related_channels : Array(AboutRelatedChannel)
property tabs : Array(String)
end
struct AboutRelatedChannel
include DB::Serializable
property ucid : String
property author : String
property author_url : String
property author_thumbnail : String
end
def get_about_info(ucid, locale)
begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@ -64,7 +35,7 @@ def get_about_info(ucid, locale)
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s }
related_channels = [] of AboutRelatedChannel
related_channels = [] of YouTubeStructs::AboutRelatedChannel
else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
@ -109,14 +80,14 @@ def get_about_info(ucid, locale)
related_author_thumbnail ||= ""
end
AboutRelatedChannel.new({
YouTubeStructs::AboutRelatedChannel.new({
ucid: related_id,
author: related_title,
author_url: related_author_url,
author_thumbnail: related_author_thumbnail,
})
end
related_channels ||= [] of AboutRelatedChannel
related_channels ||= [] of YouTubeStructs::AboutRelatedChannel
end
total_views = 0_i64
@ -155,7 +126,7 @@ def get_about_info(ucid, locale)
sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0
AboutChannel.new({
YouTubeStructs::AboutChannel.new({
ucid: ucid,
author: author,
auto_generated: auto_generated,

View file

@ -1,112 +1,3 @@
struct InvidiousChannel
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "viewCount", self.views
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
to_xml(locale, xml)
else
XML.build do |xml|
to_xml(locale, xml)
end
end
end
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| var.name }}}
}
{% end %}
end
end
class ChannelRedirect < Exception
property channel_id : String
@ -152,7 +43,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
end
def get_channel(id, db, refresh = true, pull_all_videos = true)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousStructs::Channel)
if refresh && Time.utc - channel.updated > 10.minutes
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a
@ -224,7 +115,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
video = InvidiousStructs::ChannelVideo.new({
id: video_id,
title: title,
published: published,
@ -265,7 +156,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
videos = extract_videos(initial_data, author, ucid)
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
videos = videos.map { |video| InvidiousStructs::ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
@ -299,7 +190,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
end
end
channel = InvidiousChannel.new({
channel = InvidiousStructs::Channel.new({
id: ucid,
author: author,
updated: Time.utc,

View file

@ -4,9 +4,9 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuationItems
return [] of YouTubeStructs::Renderer, nil if !continuationItems
items = [] of SearchItem
items = [] of YouTubeStructs::Renderer
continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t }
}
@ -28,7 +28,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
response = YT_POOL.client &.get(url)
initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data
return [] of YouTubeStructs::Renderer, nil if !initial_data
items = extract_items(initial_data, author, ucid)
continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?

View file

@ -65,7 +65,7 @@ def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by =
end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo
videos = [] of YouTubeStructs::VideoRenderer
2.times do |i|
initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)

View file

@ -1,11 +1,10 @@
# Data structs used by Invidious to provide certain features.
module InvidiousStructs
# Struct for representing a cached YouTube channel.
#
# This is constructed from YouTube's RSS feeds for channels and is
# currently only used for storing subscriptions in a user.
struct InvidiousChannel
struct Channel
include DB::Serializable
property id : String

View file

@ -1,5 +1,6 @@
module InvidiousStructs
private module PlaylistPrivacyConverter
# Converter to parse a Invidious privacy type string to enum
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
@ -16,7 +17,7 @@ module InvidiousStructs
property created : Time
property updated : Time
@[DB::Field(converter: PlaylistPrivacyConverter)]
@[DB::Field(converter: InvidiousStructs::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@ -25,7 +26,7 @@ module InvidiousStructs
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "type", "InvidiousStructs::Playlist"
json.field "title", self.title
json.field "playlistId", self.id

View file

@ -14,7 +14,6 @@ module YouTubeStructs
property author_thumbnail : String
property banner : String?
property description_html : String
property paid : Bool
property total_views : Int64
property sub_count : Int32
property joined : Time

View file

@ -37,12 +37,12 @@ module YouTubeStructs
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "title", video[:title]
json.field "videoId", video[:id]
json.field "lengthSeconds", video[:length_seconds]
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
generate_thumbnails(json, video[:id])
end
end
end

View file

@ -20,7 +20,6 @@ module YouTubeStructs
property description_html : String
property length_seconds : Int32
property live_now : Bool
property paid : Bool
property premium : Bool
property premiere_timestamp : Time?
@ -101,7 +100,6 @@ module YouTubeStructs
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming

View file

@ -1,10 +1,31 @@
module YouTubeStructs
# Converter to serialize first level JSON data as methods for the videos struct
module VideoJSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
# Represents an watchable video in Invidious
#
# The video struct only takes three parameters:
# - ID: The video ID
#
# - Info:
# YT Video information (streams, captions, tiles, etc). This is then serialized
# into individual properties that either stores top level stuff or accesses
# further nested data.
#
# - Updated:
# A record of when the specific struct was created and inserted
# into the DB. This is then used to measure when to cache (or update)
# videos within the database.
struct Video
include DB::Serializable
property id : String
@[DB::Field(converter: Video::JSONConverter)]
@[DB::Field(converter: YouTubeStructs::VideoJSONConverter)]
property info : Hash(String, JSON::Any)
property updated : Time
@ -20,12 +41,6 @@ module YouTubeStructs
@[DB::Field(ignore: true)]
property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
@ -277,10 +292,6 @@ module YouTubeStructs
info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def cookie
info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
end
def allow_ratings
r = info["videoDetails"]["allowRatings"]?.try &.as_bool
r.nil? ? false : r
@ -516,8 +527,13 @@ module YouTubeStructs
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
end
def is_vr : Bool
info["streamingData"]?.try &.["adaptiveFormats"].as_a[0]?.try &.["projectionType"].as_s == "MESH" ? true : false || false
def is_vr : Bool?
projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
def wilson_score : Float64
@ -531,9 +547,5 @@ module YouTubeStructs
def reason : String?
info["reason"]?.try &.as_s
end
def session_token : String?
info["sessionToken"]?.try &.as_s?
end
end
end

View file

@ -25,7 +25,7 @@ record AuthorFallback, name : String, id : String
# data is passed to the private `#parse()` method which returns a datastruct of the given
# type. Otherwise, nil is returned.
private module Parsers
# Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer
# Parses a InnerTube videoRenderer into a YouTubeStructs::VideoRenderer. Returns nil when the given object isn't a videoRenderer
#
# A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not**
# the watchable video itself.
@ -115,7 +115,7 @@ private module Parsers
end
end
# Parses a InnerTube channelRenderer into a SearchChannel. Returns nil when the given object isn't a channelRenderer
# Parses a InnerTube channelRenderer into a YouTubeStructs::ChannelRenderer. Returns nil when the given object isn't a channelRenderer
#
# A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not**
# the channel page itself.
@ -161,7 +161,7 @@ private module Parsers
end
end
# Parses a InnerTube gridPlaylistRenderer into a SearchPlaylist. Returns nil when the given object isn't a gridPlaylistRenderer
# Parses a InnerTube gridPlaylistRenderer into a YouTubeStructs::PlaylistRenderer. Returns nil when the given object isn't a gridPlaylistRenderer
#
# A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI.
# It is **not** the playlist itself.
@ -196,7 +196,7 @@ private module Parsers
end
end
# Parses a InnerTube playlistRenderer into a SearchPlaylist. Returns nil when the given object isn't a playlistRenderer
# Parses a InnerTube playlistRenderer into a YouTubeStructs::PlaylistRenderer. Returns nil when the given object isn't a playlistRenderer
#
# A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself.
#
@ -536,7 +536,7 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
# The end result is an array of YouTubeStructs::Renderer.
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) : Array(YouTubeStructs::Renderer)
items = [] of YouTubeStructs::Renderer
@ -565,27 +565,19 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
return items
end
# Flattens all items from extracted items into a one dimensional array
def flatten_items(items, target = nil)
if target.nil?
target = [] of YouTubeStructs::Renderer
end
# Extracts videos (videoRenderer) from initial InnerTube response.
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
items.each do |i|
target = [] of YouTubeStructs::Renderer
extracted.each do |i|
if i.is_a?(YouTubeStructs::Category)
target = target += i.extract_renderers
target += i.extract_renderers
else
target << i
end
end
return target
end
# Extracts videos (videoRenderer) from initial InnerTube response.
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
target = flatten_items(extracted)
return target.select(&.is_a?(YouTubeStructs::VideoRenderer)).map(&.as(YouTubeStructs::VideoRenderer))
end

View file

@ -15,13 +15,6 @@ struct SessionId
property issued : String
end
struct Annotation
include DB::Serializable
property id : String
property annotations : String
end
struct ConfigPreferences
include YAML::Serializable
@ -444,7 +437,7 @@ def create_notification_stream(env, topics, connection_channel)
case topic
when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video|
topic, Time.unix(since.not_nil!), as: InvidiousStructs::ChannelVideo).each do |video|
response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?

View file

@ -1,256 +0,0 @@
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
xml.element("media:community") do
xml.element("media:statistics", views: self.views)
end
end
end
def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def is_upcoming
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
class Category
include DB::Serializable
property title : String
property contents : Array(SearchItem) | Array(Video)
property url : String?
property description_html : String
property badges : Array(Tuple(String, String))?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "title", self.title
json.field "contents", self.contents
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

View file

@ -6,7 +6,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC
SQL
POPULAR_VIDEOS = Atomic.new([] of ChannelVideo)
POPULAR_VIDEOS = Atomic.new([] of InvidiousStructs::ChannelVideo)
private getter db : DB::Database
def initialize(@db)
@ -14,7 +14,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
def begin
loop do
videos = db.query_all(QUERY, as: ChannelVideo)
videos = db.query_all(QUERY, as: InvidiousStructs::ChannelVideo)
.sort_by(&.published)
.reverse

View file

@ -26,7 +26,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
begin
# Drop outdated views
column_array = get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i|
InvidiousStructs::ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}")

View file

@ -1,4 +1,4 @@
struct PlaylistVideo
struct YouTubeStructs::PlaylistVideo
include DB::Serializable
property title : String
@ -92,181 +92,16 @@ struct PlaylistVideo
end
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if !offset || offset == 0
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json, offset + index)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
end
end
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description).gsub("\n", "<br>")
end
end
def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
playlist = InvidiousStructs::Playlist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
@ -287,7 +122,7 @@ def create_playlist(db, title, privacy, user)
end
def subscribe_playlist(db, user, playlist)
playlist = InvidiousPlaylist.new({
playlist = InvidiousStructs::Playlist.new({
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
@ -346,7 +181,7 @@ end
def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
if plid.starts_with? "IV"
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
return playlist
else
raise InfoException.new("Playlist does not exist.")
@ -411,7 +246,7 @@ def fetch_playlist(plid, locale)
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
end
return Playlist.new({
return YouTubeStructs::Playlist.new({
title: title,
id: plid,
author: author,
@ -430,12 +265,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
# Show empy playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
return [] of YouTubeStructs::PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
if playlist.is_a? InvidiousStructs::Playlist
db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3",
playlist.id, playlist.index, offset, as: PlaylistVideo)
playlist.id, playlist.index, offset, as: YouTubeStructs::PlaylistVideo)
else
if offset >= 100
# Normalize offset to match youtube's behavior (100 videos chunck per request)
@ -452,7 +287,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo
videos = [] of YouTubeStructs::PlaylistVideo
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@ -493,7 +328,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
length_seconds = 0
end
videos << PlaylistVideo.new({
videos << YouTubeStructs::PlaylistVideo.new({
title: title,
id: video_id,
author: author,

View file

@ -78,7 +78,7 @@ module Invidious::Routes::API::V1::Authenticated
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel)
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel)
JSON.build do |json|
json.array do
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated
env.response.content_type = "application/json"
user = env.get("user").as(User)
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist)
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousStructs::Playlist)
JSON.build do |json|
json.array do
@ -174,7 +174,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -207,7 +207,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -230,7 +230,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end
@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(500, ex)
end
playlist_video = PlaylistVideo.new({
playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
@ -286,7 +286,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]
index = env.params.url["index"].to_i64(16)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
end

View file

@ -19,7 +19,7 @@ module Invidious::Routes::API::V1::Channels
page = 1
if channel.auto_generated
videos = [] of SearchVideo
videos = [] of YouTubeStructs::VideoRenderer
count = 0
else
begin
@ -208,7 +208,7 @@ module Invidious::Routes::API::V1::Channels
json.field "playlists" do
json.array do
items.each do |item|
item.to_json(locale, json) if item.is_a?(SearchPlaylist)
item.to_json(locale, json) if item.is_a?(YouTubeStructs::PlaylistRenderer)
end
end
end

View file

@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos
case source
when "archive"
if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation))
if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: YouTubeStructs::Annotation))
annotations = cached_annotation.annotations
else
index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')

View file

@ -29,7 +29,7 @@ module Invidious::Routes::Channels
item.author
end
end
items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist))
items = items.select(&.is_a?(YouTubeStructs::PlaylistRenderer)).map(&.as(YouTubeStructs::PlaylistRenderer))
items.each { |item| item.author = "" }
else
sort_options = {"newest", "oldest", "popular"}
@ -57,7 +57,7 @@ module Invidious::Routes::Channels
end
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) }
items = items.select { |item| item.is_a?(YouTubeStructs::PlaylistRenderer) }.map { |item| item.as(YouTubeStructs::PlaylistRenderer) }
items.each { |item| item.author = "" }
templated "playlists"

View file

@ -15,13 +15,13 @@ module Invidious::Routes::Feeds
user = user.as(User)
items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
items_created.map! do |item|
item.author = ""
item
end
items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist)
items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
items_saved.map! do |item|
item.author = ""
item
@ -169,7 +169,7 @@ module Invidious::Routes::Feeds
description_html = entry.xpath_node("group/description").not_nil!.to_s
views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
SearchVideo.new({
YouTubeStructs::VideoRenderer.new({
title: title,
id: video_id,
author: author,
@ -264,7 +264,7 @@ module Invidious::Routes::Feeds
path = env.request.path
if plid.starts_with? "IV"
if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale)
return XML.build(indent: " ", encoding: "UTF-8") do |xml|
@ -405,7 +405,7 @@ module Invidious::Routes::Feeds
}.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'")
video = ChannelVideo.new({
video = InvidiousStructs::ChannelVideo.new({
id: id,
title: video.title,
published: published,

View file

@ -85,7 +85,7 @@ module Invidious::Routes::Playlists
sid = sid.as(String)
plid = env.params.query["list"]?
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -117,7 +117,7 @@ module Invidious::Routes::Playlists
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -149,7 +149,7 @@ module Invidious::Routes::Playlists
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -160,7 +160,7 @@ module Invidious::Routes::Playlists
begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
rescue ex
videos = [] of PlaylistVideo
videos = [] of YouTubeStructs::PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB)
@ -190,7 +190,7 @@ module Invidious::Routes::Playlists
return error_template(400, ex)
end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -233,7 +233,7 @@ module Invidious::Routes::Playlists
page ||= 1
begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email
return env.redirect referer
end
@ -245,13 +245,13 @@ module Invidious::Routes::Playlists
if query
begin
search_query, count, items, operators = process_search_query(query, page, user, region: nil)
videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) }
videos = items.select { |item| item.is_a? YouTubeStructs::VideoRenderer }.map { |item| item.as(YouTubeStructs::VideoRenderer) }
rescue ex
videos = [] of SearchVideo
videos = [] of YouTubeStructs::VideoRenderer
count = 0
end
else
videos = [] of SearchVideo
videos = [] of YouTubeStructs::VideoRenderer
count = 0
end
@ -311,7 +311,7 @@ module Invidious::Routes::Playlists
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist)
playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousStructs::Playlist)
raise "Invalid user" if playlist.author != user.email
rescue ex
if redirect
@ -351,7 +351,7 @@ module Invidious::Routes::Playlists
end
end
playlist_video = PlaylistVideo.new({
playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,

View file

@ -17,9 +17,9 @@ def channel_search(query, page, channel)
continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return 0, [] of SearchItem if !continuationItems
return 0, [] of YouTubeStructs::Renderer if !continuationItems
items = [] of SearchItem
items = [] of YouTubeStructs::Renderer
continuationItems.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0])
.try { |t| items << t }
@ -29,7 +29,7 @@ def channel_search(query, page, channel)
end
def search(query, search_params = produce_search_params(content_type: "all"), region = nil)
return 0, [] of SearchItem if query.empty?
return 0, [] of YouTubeStructs::Renderer if query.empty?
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
@ -219,10 +219,10 @@ def process_search_query(query, page, user, region)
to_tsvector(#{view_name}.author)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: InvidiousStructs::ChannelVideo)
count = items.size
else
items = [] of ChannelVideo
items = [] of InvidiousStructs::ChannelVideo
count = 0
end
else
@ -234,14 +234,10 @@ def process_search_query(query, page, user, region)
# Light processing to flatten search results out of Categories.
# They should ideally be supported in the future.
items_without_category = [] of SearchItem | ChannelVideo
items_without_category = [] of YouTubeStructs::Renderer | InvidiousStructs::ChannelVideo
items.each do |i|
if i.is_a? Category
i.contents.each do |nest_i|
if !nest_i.is_a? Video
items_without_category << nest_i
end
end
if i.is_a? YouTubeStructs::Category
items_without_category += i.extract_renderers
else
items_without_category << i
end

View file

@ -503,8 +503,8 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
args = arg_array(notifications)
notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo)
videos = [] of ChannelVideo
notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: InvidiousStructs::ChannelVideo)
videos = [] of InvidiousStructs::ChannelVideo
notifications.sort_by! { |video| video.published }.reverse!
@ -530,11 +530,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo)
end
videos.sort_by! { |video| video.published }.reverse!
@ -547,11 +547,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo)
end
end

View file

@ -221,583 +221,6 @@ VIDEO_FORMATS = {
"397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
}
struct VideoPreferences
include JSON::Serializable
property annotations : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
property continue_autoplay : Bool
property controls : Bool
property listen : Bool
property local : Bool
property preferred_captions : Array(String)
property player_style : String
property quality : String
property quality_dash : String
property raw : Bool
property region : String?
property related_videos : Bool
property speed : Float32 | Float64
property video_end : Float64 | Int32
property video_loop : Bool
property extend_desc : Bool
property video_start : Float64 | Int32
property volume : Int32
property vr_mode : Bool
end
struct Video
include DB::Serializable
property id : String
@[DB::Field(converter: Video::JSONConverter)]
property info : Hash(String, JSON::Any)
property updated : Time
@[DB::Field(ignore: true)]
property captions : Array(Caption)?
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "error", info["reason"] if info["reason"]?
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "storyboards" do
generate_storyboards(json, self.id, self.storyboards)
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "keywords", self.keywords
json.field "viewCount", self.views
json.field "likeCount", self.likes
json.field "dislikeCount", self.dislikes
json.field "paid", self.paid
json.field "premium", self.premium
json.field "isFamilyFriendly", self.is_family_friendly
json.field "allowedRegions", self.allowed_regions
json.field "genre", self.genre
json.field "genreUrl", self.genre_url
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCountText", self.sub_count_text
json.field "lengthSeconds", self.length_seconds
json.field "allowRatings", self.allow_ratings
json.field "rating", self.average_rating
json.field "isListed", self.is_listed
json.field "liveNow", self.live_now
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
if hlsvp = self.hls_manifest_url
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
json.field "adaptiveFormats" do
json.array do
self.adaptive_fmts.each do |fmt|
json.object do
json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
json.field "bitrate", fmt["bitrate"].as_i.to_s
json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]
json.field "lmt", fmt["lastModified"]
json.field "projectionType", fmt["projectionType"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "formatStreams" do
json.array do
self.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "captions" do
json.array do
self.captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "languageCode", caption.languageCode
json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
json.field "recommendedVideos" do
json.array do
self.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
generate_thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
json.field "authorUrl", rv["author_url"]?
json.field "authorId", rv["ucid"]?
if rv["author_thumbnail"]?
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
json.field "viewCountText", rv["short_view_count_text"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def title
info["videoDetails"]["title"]?.try &.as_s || ""
end
def ucid
info["videoDetails"]["channelId"]?.try &.as_s || ""
end
def author
info["videoDetails"]["author"]?.try &.as_s || ""
end
def length_seconds : Int32
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
end
def views : Int64
info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
end
def likes : Int64
info["likes"]?.try &.as_i64 || 0_i64
end
def dislikes : Int64
info["dislikes"]?.try &.as_i64 || 0_i64
end
def average_rating : Float64
# (likes / (likes + dislikes) * 4 + 1)
info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
end
def published : Time
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def allow_ratings
r = info["videoDetails"]["allowRatings"]?.try &.as_bool
r.nil? ? false : r
end
def live_now
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
end
def is_listed
info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
end
def is_upcoming
info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
end
def premiere_timestamp : Time?
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
end
def keywords
info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
end
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
def allowed_regions
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
end
def author_thumbnail : String
info["authorThumbnail"]?.try &.as_s || ""
end
def sub_count_text : String
info["subCountText"]?.try &.as_s || "-"
end
def fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
s.each do |k, v|
fmt[k] = JSON::Any.new(v)
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@fmt_stream = fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
def adaptive_fmts
return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
s.each do |k, v|
fmt[k] = JSON::Any.new(v)
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
# See https://github.com/TeamNewPipe/NewPipe/issues/2415
# Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
fmt_stream.reject! { |f| !f["indexRange"]? }
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@adaptive_fmts = fmt_stream
return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
end
def audio_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
def storyboards
storyboards = info["storyboards"]?
.try &.as_h
.try &.["playerStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s.split("|")
if !storyboards
if storyboard = info["storyboards"]?
.try &.as_h
.try &.["playerLiveStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s
return [{
url: storyboard.split("#")[0],
width: 106,
height: 60,
count: -1,
interval: 5000,
storyboard_width: 3,
storyboard_height: 3,
storyboard_count: -1,
}]
end
end
items = [] of NamedTuple(
url: String,
width: Int32,
height: Int32,
count: Int32,
interval: Int32,
storyboard_width: Int32,
storyboard_height: Int32,
storyboard_count: Int32)
return items if !storyboards
url = URI.parse(storyboards.shift)
params = HTTP::Params.parse(url.query || "")
storyboards.each_with_index do |storyboard, i|
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#")
params["sigh"] = sigh
url.query = params.to_s
width = width.to_i
height = height.to_i
count = count.to_i
interval = interval.to_i
storyboard_width = storyboard_width.to_i
storyboard_height = storyboard_height.to_i
storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
items << {
url: url.to_s.sub("$L", i).sub("$N", "M$M"),
width: width,
height: height,
count: count,
interval: interval,
storyboard_width: storyboard_width,
storyboard_height: storyboard_height,
storyboard_count: storyboard_count,
}
end
items
end
def paid
reason = info["playabilityStatus"]?.try &.["reason"]?
paid = reason == "This video requires payment to watch." ? true : false
paid
end
def premium
keywords.includes? "YouTube Red"
end
def captions : Array(Caption)
return @captions.as(Array(Caption)) if @captions
captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
languageCode = caption["languageCode"].to_s
baseUrl = caption["baseUrl"].to_s
caption = Caption.new(name.to_s, languageCode, baseUrl)
caption.name = caption.name.split(" - ")[0]
caption
end
captions ||= [] of Caption
@captions = captions
return @captions.as(Array(Caption))
end
def description
description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
end
# TODO
def description=(value : String)
@description = value
end
def description_html
info["descriptionHtml"]?.try &.as_s || "<p></p>"
end
def description_html=(value : String)
info["descriptionHtml"] = JSON::Any.new(value)
end
def short_description
info["shortDescription"]?.try &.as_s? || ""
end
def hls_manifest_url : String?
info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s
end
def dash_manifest_url
info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
end
def genre : String
info["genre"]?.try &.as_s || ""
end
def genre_url : String?
info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
def license : String?
info["license"]?.try &.as_s
end
def is_family_friendly : Bool
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
end
def is_vr : Bool?
projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
def wilson_score : Float64
ci_lower_bound(likes, likes + dislikes).round(4)
end
def engagement : Float64
(((likes + dislikes) / views) * 100).round(4)
end
def reason : String?
info["reason"]?.try &.as_s
end
end
struct Caption
property name
property languageCode
property baseUrl
getter name : String
getter languageCode : String
getter baseUrl : String
setter name
def initialize(@name, @languageCode, @baseUrl)
end
end
class VideoRedirect < Exception
property video_id : String
@ -942,7 +365,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
end
def get_video(id, db, refresh = true, region = nil, force_refresh = false)
if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region
if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: YouTubeStructs::Video)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
if (refresh &&
@ -967,6 +390,8 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false)
return video
end
# TODO make private. All instances of fetching video should be done from get_video() to
# allow for caching.
def fetch_video(id, region)
info = extract_video_info(video_id: id)
@ -993,7 +418,7 @@ def fetch_video(id, region)
raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]?
video = Video.new({
video = YouTubeStructs::Video.new({
id: id,
info: info,
updated: Time.utc,
@ -1116,7 +541,7 @@ def process_video_params(query, preferences)
controls ||= 1
controls = controls >= 1
params = VideoPreferences.new({
params = InvidiousStructs::VideoPreferences.new({
annotations: annotations,
autoplay: autoplay,
comments: comments,

View file

@ -1,7 +1,7 @@
<div class="pure-u-1 pure-u-md-1-4">
<div class="h-box">
<% case item when %>
<% when SearchChannel %>
<% when YouTubeStructs::ChannelRenderer %>
<a href="/channel/<%= item.ucid %>">
<% if !env.get("preferences").as(Preferences).thin_mode %>
<center>
@ -13,7 +13,7 @@
<p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p>
<% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %>
<h5><%= item.description_html %></h5>
<% when SearchPlaylist, InvidiousPlaylist %>
<% when YouTubeStructs::PlaylistRenderer, InvidiousStructs::Playlist %>
<% if item.id.starts_with? "RD" %>
<% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %>
<% else %>
@ -47,7 +47,7 @@
<a href="/channel/<%= item.ucid %>">
<p dir="auto"><b><%= HTML.escape(item.author) %></b></p>
</a>
<% when PlaylistVideo %>
<% when YouTubeStructs::PlaylistVideo %>
<a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>">
<% if !env.get("preferences").as(Preferences).thin_mode %>
<div class="thumbnail">
@ -109,7 +109,7 @@
</div>
<% end %>
</div>
<% when Category %>
<% when YouTubeStructs::Category %>
<% else %>
<a style="width:100%" href="/watch?v=<%= item.id %>">
<% if !env.get("preferences").as(Preferences).thin_mode %>

View file

@ -44,7 +44,7 @@
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>">
</form>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="h-box" style="text-align:right">
<h3>
<a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a>

View file

@ -9,7 +9,7 @@
<div class="pure-g h-box">
<div class="pure-u-2-3">
<h3><%= title %></h3>
<% if playlist.is_a? InvidiousPlaylist %>
<% if playlist.is_a? InvidiousStructs::Playlist %>
<b>
<% if playlist.author == user.try &.email %>
<a href="/feed/playlists"><%= author %></a> |
@ -18,7 +18,7 @@
<% end %>
<%= translate(locale, "`x` videos", "#{playlist.video_count}") %> |
<%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> |
<% case playlist.as(InvidiousPlaylist).privacy when %>
<% case playlist.as(InvidiousStructs::Playlist).privacy when %>
<% when PlaylistPrivacy::Public %>
<i class="icon ion-md-globe"></i> <%= translate(locale, "Public") %>
<% when PlaylistPrivacy::Unlisted %>
@ -35,7 +35,7 @@
</b>
<% end %>
<% if !playlist.is_a? InvidiousPlaylist %>
<% if !playlist.is_a? InvidiousStructs::Playlist %>
<div class="pure-u-2-3">
<a href="https://www.youtube.com/playlist?list=<%= playlist.id %>">
<%= translate(locale, "View playlist on YouTube") %>
@ -50,7 +50,7 @@
<div class="pure-u-1-3" style="text-align:right">
<h3>
<div class="pure-g user-field">
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
<% else %>
@ -72,7 +72,7 @@
</div>
</div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="h-box" style="text-align:right">
<h3>
<a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a>
@ -84,7 +84,7 @@
<hr>
</div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %>
<% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<script id="playlist_data" type="application/json">
<%=
{