mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-03-16.git
synced 2024-08-15 00:53:18 +00:00
Restructure data_structs and begin migrations
This commit is contained in:
syeopite
parent
3c01bbb0b3
commit
2333221e14
18 changed files with 938 additions and 102 deletions
|
|
@ -30,6 +30,7 @@ require "./invidious/helpers/*"
|
|||
require "./invidious/*"
|
||||
require "./invidious/channels/*"
|
||||
require "./invidious/routes/**"
|
||||
require "./invidious/data_structs/**"
|
||||
require "./invidious/jobs/**"
|
||||
|
||||
CONFIG = Config.load
|
||||
|
|
|
|||
3
src/invidious/data_structs/base.cr
Normal file
3
src/invidious/data_structs/base.cr
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module YouTubeStructs
|
||||
alias Renderer = Category | VideoRenderer | PlaylistRenderer | ChannelRenderer
|
||||
end
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
|
||||
# Data structs used by Invidious to provide certain features.
|
||||
module InvidiousStructs
|
||||
# Struct for representing a cached Invidious channel.
|
||||
# Struct for representing a cached YouTube channel.
|
||||
#
|
||||
# Currently used for storing subscriptions.
|
||||
# This is constructed from YouTube's RSS feeds for channels and is
|
||||
# currently only used for storing subscriptions in a user.
|
||||
struct InvidiousChannel
|
||||
include DB::Serializable
|
||||
|
||||
|
|
@ -9,11 +12,20 @@ module InvidiousStructs
|
|||
property author : String
|
||||
property updated : Time
|
||||
property deleted : Bool
|
||||
# TODO I don't believe the subscripted attribute is actually used.
|
||||
# so this can likely be removed.
|
||||
property subscribed : Time?
|
||||
end
|
||||
end
|
||||
|
||||
module YTStructs
|
||||
# Struct for representing a video from a YouTube channel
|
||||
#
|
||||
# This is constructed from YouTube's RSS feeds for channels and is
|
||||
# used for referencing videos used by Invidious exclusive features. IE popular feeds,
|
||||
# notifications, subscriptions, etc.
|
||||
#
|
||||
# TODO ideally this should be expanded to include all channel videos. That way
|
||||
# we can implement optional caching of YT requests in a DB such as redis.
|
||||
#
|
||||
struct ChannelVideo
|
||||
include DB::Serializable
|
||||
|
||||
92
src/invidious/data_structs/invidious/playlists.cr
Normal file
92
src/invidious/data_structs/invidious/playlists.cr
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
module InvidiousStructs
|
||||
private module PlaylistPrivacyConverter
|
||||
def self.from_rs(rs)
|
||||
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
|
||||
end
|
||||
end
|
||||
|
||||
struct Playlist
|
||||
include DB::Serializable
|
||||
|
||||
property title : String
|
||||
property id : String
|
||||
property author : String
|
||||
property description : String = ""
|
||||
property video_count : Int32
|
||||
property created : Time
|
||||
property updated : Time
|
||||
|
||||
@[DB::Field(converter: PlaylistPrivacyConverter)]
|
||||
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
|
||||
property index : Array(Int64)
|
||||
|
||||
@[DB::Field(ignore: true)]
|
||||
property thumbnail_id : String?
|
||||
|
||||
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
|
||||
json.object do
|
||||
json.field "type", "invidiousPlaylist"
|
||||
json.field "title", self.title
|
||||
json.field "playlistId", self.id
|
||||
|
||||
json.field "author", self.author
|
||||
json.field "authorId", self.ucid
|
||||
json.field "authorUrl", nil
|
||||
json.field "authorThumbnails", [] of String
|
||||
|
||||
json.field "description", html_to_content(self.description_html)
|
||||
json.field "descriptionHtml", self.description_html
|
||||
json.field "videoCount", self.video_count
|
||||
|
||||
json.field "viewCount", self.views
|
||||
json.field "updated", self.updated.to_unix
|
||||
json.field "isListed", self.privacy.public?
|
||||
|
||||
json.field "videos" do
|
||||
json.array do
|
||||
if !offset || offset == 0
|
||||
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64)
|
||||
offset = self.index.index(index) || 0
|
||||
end
|
||||
|
||||
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
|
||||
videos.each_with_index do |video, index|
|
||||
video.to_json(locale, json, offset + index)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
|
||||
if json
|
||||
to_json(offset, locale, json, continuation: continuation)
|
||||
else
|
||||
JSON.build do |json|
|
||||
to_json(offset, locale, json, continuation: continuation)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def thumbnail
|
||||
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
|
||||
"/vi/#{@thumbnail_id}/mqdefault.jpg"
|
||||
end
|
||||
|
||||
def author_thumbnail
|
||||
nil
|
||||
end
|
||||
|
||||
def ucid
|
||||
nil
|
||||
end
|
||||
|
||||
def views
|
||||
0_i64
|
||||
end
|
||||
|
||||
def description_html
|
||||
HTML.escape(self.description).gsub("\n", "<br>")
|
||||
end
|
||||
end
|
||||
end
|
||||
29
src/invidious/data_structs/invidious/video_preferences.cr
Normal file
29
src/invidious/data_structs/invidious/video_preferences.cr
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
module InvidiousStructs
|
||||
# Struct containing all values for video preferences
|
||||
struct VideoPreferences
|
||||
include JSON::Serializable
|
||||
|
||||
property annotations : Bool
|
||||
property autoplay : Bool
|
||||
property comments : Array(String)
|
||||
property continue : Bool
|
||||
property continue_autoplay : Bool
|
||||
property controls : Bool
|
||||
property listen : Bool
|
||||
property local : Bool
|
||||
property preferred_captions : Array(String)
|
||||
property player_style : String
|
||||
property quality : String
|
||||
property quality_dash : String
|
||||
property raw : Bool
|
||||
property region : String?
|
||||
property related_videos : Bool
|
||||
property speed : Float32 | Float64
|
||||
property video_end : Float64 | Int32
|
||||
property video_loop : Bool
|
||||
property extend_desc : Bool
|
||||
property video_start : Float64 | Int32
|
||||
property volume : Int32
|
||||
property vr_mode : Bool
|
||||
end
|
||||
end
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
module YTStructs
|
||||
alias Renderers = VideoRenderer | ChannelRenderer | PlaylistRenderer | Category
|
||||
|
||||
# Wrapper object around all renderer types
|
||||
struct AnyRenderer
|
||||
def initialize(@raw : Renderers)
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `VideoRenderer`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_video
|
||||
@raw.as(VideoRenderer)
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `VideoRenderer`, and returns its value.
|
||||
# Returns `nil` otherwise
|
||||
def as_video?
|
||||
as_video if @raw.is_a? VideoRenderer
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `ChannelRenderer`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_channel
|
||||
@raw.as(ChannelRenderer)
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `ChannelRenderer`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_channel?
|
||||
as_channel if @raw.is_a? ChannelRenderer
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `PlaylistRenderer`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_playlist
|
||||
@raw.as(PlaylistRenderer)
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `PlaylistRenderer`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_playlist?
|
||||
as_playlist if @raw.is_a? PlaylistRenderer
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `Category`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_category
|
||||
@raw.as(Category)
|
||||
end
|
||||
|
||||
# Checks that the underlying value is `Category`, and returns its value.
|
||||
# Raises otherwise
|
||||
def as_category?
|
||||
as_category if @raw.is_a? Category
|
||||
end
|
||||
end
|
||||
end
|
||||
9
src/invidious/data_structs/youtube/annotations.cr
Normal file
9
src/invidious/data_structs/youtube/annotations.cr
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
module YouTubeStructs
|
||||
struct Annotation
|
||||
include DB::Serializable
|
||||
|
||||
property id : String
|
||||
# JSON String containing annotation data
|
||||
property annotations : String
|
||||
end
|
||||
end
|
||||
16
src/invidious/data_structs/youtube/caption.cr
Normal file
16
src/invidious/data_structs/youtube/caption.cr
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
module YouTubeStructs
|
||||
struct Caption
|
||||
property name
|
||||
property languageCode
|
||||
property baseUrl
|
||||
|
||||
getter name : String
|
||||
getter languageCode : String
|
||||
getter baseUrl : String
|
||||
|
||||
setter name
|
||||
|
||||
def initialize(@name, @languageCode, @baseUrl)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module YTStructs
|
||||
module YouTubeStructs
|
||||
# Struct to represent channel heading information.
|
||||
#
|
||||
# As of master this is mostly taken from the about tab.
|
||||
98
src/invidious/data_structs/youtube/playlist_videos.cr
Normal file
98
src/invidious/data_structs/youtube/playlist_videos.cr
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
module YouTubeStructs
|
||||
# Represents a video within a playlist.
|
||||
#
|
||||
# TODO Make consistent with VideoRenderer. Maybe inherit from abstract struct?
|
||||
struct PlaylistVideo
|
||||
include DB::Serializable
|
||||
|
||||
property title : String
|
||||
property id : String
|
||||
property author : String
|
||||
property ucid : String
|
||||
property length_seconds : Int32
|
||||
property published : Time
|
||||
property plid : String
|
||||
property index : Int64
|
||||
property live_now : Bool
|
||||
|
||||
def to_xml(auto_generated, xml : XML::Builder)
|
||||
xml.element("entry") do
|
||||
xml.element("id") { xml.text "yt:video:#{self.id}" }
|
||||
xml.element("yt:videoId") { xml.text self.id }
|
||||
xml.element("yt:channelId") { xml.text self.ucid }
|
||||
xml.element("title") { xml.text self.title }
|
||||
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
|
||||
|
||||
xml.element("author") do
|
||||
if auto_generated
|
||||
xml.element("name") { xml.text self.author }
|
||||
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
|
||||
else
|
||||
xml.element("name") { xml.text author }
|
||||
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
|
||||
end
|
||||
end
|
||||
|
||||
xml.element("content", type: "xhtml") do
|
||||
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
|
||||
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
|
||||
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
|
||||
|
||||
xml.element("media:group") do
|
||||
xml.element("media:title") { xml.text self.title }
|
||||
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
|
||||
width: "320", height: "180")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_xml(auto_generated, xml : XML::Builder? = nil)
|
||||
if xml
|
||||
to_xml(auto_generated, xml)
|
||||
else
|
||||
XML.build do |json|
|
||||
to_xml(auto_generated, xml)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(locale, json : JSON::Builder, index : Int32?)
|
||||
json.object do
|
||||
json.field "title", self.title
|
||||
json.field "videoId", self.id
|
||||
|
||||
json.field "author", self.author
|
||||
json.field "authorId", self.ucid
|
||||
json.field "authorUrl", "/channel/#{self.ucid}"
|
||||
|
||||
json.field "videoThumbnails" do
|
||||
generate_thumbnails(json, self.id)
|
||||
end
|
||||
|
||||
if index
|
||||
json.field "index", index
|
||||
json.field "indexId", self.index.to_u64.to_s(16).upcase
|
||||
else
|
||||
json.field "index", self.index
|
||||
end
|
||||
|
||||
json.field "lengthSeconds", self.length_seconds
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
|
||||
if json
|
||||
to_json(locale, json, index: index)
|
||||
else
|
||||
JSON.build do |json|
|
||||
to_json(locale, json, index: index)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
75
src/invidious/data_structs/youtube/playlists.cr
Normal file
75
src/invidious/data_structs/youtube/playlists.cr
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
module YouTubeStructs
|
||||
struct Playlist
|
||||
include DB::Serializable
|
||||
|
||||
property title : String
|
||||
property id : String
|
||||
property author : String
|
||||
property author_thumbnail : String
|
||||
property ucid : String
|
||||
property description : String
|
||||
property description_html : String
|
||||
property video_count : Int32
|
||||
property views : Int64
|
||||
property updated : Time
|
||||
property thumbnail : String?
|
||||
|
||||
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
|
||||
json.object do
|
||||
json.field "type", "playlist"
|
||||
json.field "title", self.title
|
||||
json.field "playlistId", self.id
|
||||
json.field "playlistThumbnail", self.thumbnail
|
||||
|
||||
json.field "author", self.author
|
||||
json.field "authorId", self.ucid
|
||||
json.field "authorUrl", "/channel/#{self.ucid}"
|
||||
|
||||
json.field "authorThumbnails" do
|
||||
json.array do
|
||||
qualities = {32, 48, 76, 100, 176, 512}
|
||||
|
||||
qualities.each do |quality|
|
||||
json.object do
|
||||
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
|
||||
json.field "width", quality
|
||||
json.field "height", quality
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "description", self.description
|
||||
json.field "descriptionHtml", self.description_html
|
||||
json.field "videoCount", self.video_count
|
||||
|
||||
json.field "viewCount", self.views
|
||||
json.field "updated", self.updated.to_unix
|
||||
json.field "isListed", self.privacy.public?
|
||||
|
||||
json.field "videos" do
|
||||
json.array do
|
||||
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
|
||||
videos.each_with_index do |video, index|
|
||||
video.to_json(locale, json)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
|
||||
if json
|
||||
to_json(offset, locale, json, continuation: continuation)
|
||||
else
|
||||
JSON.build do |json|
|
||||
to_json(offset, locale, json, continuation: continuation)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def privacy
|
||||
PlaylistPrivacy::Public
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module YTStructs
|
||||
module YouTubeStructs
|
||||
# Struct to represent an InnerTube `"shelfRenderers"`
|
||||
#
|
||||
# A shelfRenderer renders divided sections on YouTube. IE "People also watched" in search results and
|
||||
|
|
@ -13,11 +13,20 @@ module YTStructs
|
|||
include DB::Serializable
|
||||
|
||||
property title : String
|
||||
property contents : Array(SearchItem) | Array(Video)
|
||||
property contents : Array(Renderer) | Array(Video)
|
||||
property url : String?
|
||||
property description_html : String
|
||||
property badges : Array(Tuple(String, String))?
|
||||
|
||||
# Extracts all renderers out of the category's contents.
|
||||
def extract_renderers()
|
||||
target = [] of Renderer
|
||||
|
||||
@contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
|
||||
|
||||
return target
|
||||
end
|
||||
|
||||
def to_json(locale, json : JSON::Builder)
|
||||
json.object do
|
||||
json.field "title", self.title
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module YTStructs
|
||||
module YouTubeStructs
|
||||
# Struct to represent an InnerTube `"channelRenderer"`
|
||||
#
|
||||
# A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not**
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
module YTStructs
|
||||
alias PlaylistRendererVideo = NamedTuple(title: String, id: String, length_seconds: Int32)
|
||||
module YouTubeStructs
|
||||
alias PlaylistVideoRenderer = NamedTuple(title: String, id: String, length_seconds: Int32)
|
||||
|
||||
# Struct to represent an InnerTube `"PlaylistRenderer"`
|
||||
#
|
||||
|
|
@ -18,7 +18,7 @@ module YTStructs
|
|||
property author : String
|
||||
property ucid : String
|
||||
property video_count : Int32
|
||||
property videos : Array(PlaylistRendererVideo)
|
||||
property videos : Array(PlaylistVideoRenderer)
|
||||
property thumbnail : String?
|
||||
|
||||
def to_json(locale, json : JSON::Builder)
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module YTStructs
|
||||
module YouTubeStructs
|
||||
# Struct to represent an InnerTube `"videoRenderer"`
|
||||
#
|
||||
# A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not**
|
||||
539
src/invidious/data_structs/youtube/videos.cr
Normal file
539
src/invidious/data_structs/youtube/videos.cr
Normal file
|
|
@ -0,0 +1,539 @@
|
|||
module YouTubeStructs
|
||||
struct Video
|
||||
include DB::Serializable
|
||||
|
||||
property id : String
|
||||
|
||||
@[DB::Field(converter: Video::JSONConverter)]
|
||||
property info : Hash(String, JSON::Any)
|
||||
property updated : Time
|
||||
|
||||
@[DB::Field(ignore: true)]
|
||||
property captions : Array(Caption)?
|
||||
|
||||
@[DB::Field(ignore: true)]
|
||||
property adaptive_fmts : Array(Hash(String, JSON::Any))?
|
||||
|
||||
@[DB::Field(ignore: true)]
|
||||
property fmt_stream : Array(Hash(String, JSON::Any))?
|
||||
|
||||
@[DB::Field(ignore: true)]
|
||||
property description : String?
|
||||
|
||||
module JSONConverter
|
||||
def self.from_rs(rs)
|
||||
JSON.parse(rs.read(String)).as_h
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
|
||||
json.object do
|
||||
json.field "type", "video"
|
||||
|
||||
json.field "title", self.title
|
||||
json.field "videoId", self.id
|
||||
|
||||
json.field "error", info["reason"] if info["reason"]?
|
||||
|
||||
json.field "videoThumbnails" do
|
||||
generate_thumbnails(json, self.id)
|
||||
end
|
||||
json.field "storyboards" do
|
||||
generate_storyboards(json, self.id, self.storyboards)
|
||||
end
|
||||
|
||||
json.field "description", self.description
|
||||
json.field "descriptionHtml", self.description_html
|
||||
json.field "published", self.published.to_unix
|
||||
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
|
||||
json.field "keywords", self.keywords
|
||||
|
||||
json.field "viewCount", self.views
|
||||
json.field "likeCount", self.likes
|
||||
json.field "dislikeCount", self.dislikes
|
||||
|
||||
json.field "paid", self.paid
|
||||
json.field "premium", self.premium
|
||||
json.field "isFamilyFriendly", self.is_family_friendly
|
||||
json.field "allowedRegions", self.allowed_regions
|
||||
json.field "genre", self.genre
|
||||
json.field "genreUrl", self.genre_url
|
||||
|
||||
json.field "author", self.author
|
||||
json.field "authorId", self.ucid
|
||||
json.field "authorUrl", "/channel/#{self.ucid}"
|
||||
|
||||
json.field "authorThumbnails" do
|
||||
json.array do
|
||||
qualities = {32, 48, 76, 100, 176, 512}
|
||||
|
||||
qualities.each do |quality|
|
||||
json.object do
|
||||
json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
|
||||
json.field "width", quality
|
||||
json.field "height", quality
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "subCountText", self.sub_count_text
|
||||
|
||||
json.field "lengthSeconds", self.length_seconds
|
||||
json.field "allowRatings", self.allow_ratings
|
||||
json.field "rating", self.average_rating
|
||||
json.field "isListed", self.is_listed
|
||||
json.field "liveNow", self.live_now
|
||||
json.field "isUpcoming", self.is_upcoming
|
||||
|
||||
if self.premiere_timestamp
|
||||
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
|
||||
end
|
||||
|
||||
if hlsvp = self.hls_manifest_url
|
||||
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
|
||||
json.field "hlsUrl", hlsvp
|
||||
end
|
||||
|
||||
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
|
||||
|
||||
json.field "adaptiveFormats" do
|
||||
json.array do
|
||||
self.adaptive_fmts.each do |fmt|
|
||||
json.object do
|
||||
json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
|
||||
json.field "bitrate", fmt["bitrate"].as_i.to_s
|
||||
json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
|
||||
json.field "url", fmt["url"]
|
||||
json.field "itag", fmt["itag"].as_i.to_s
|
||||
json.field "type", fmt["mimeType"]
|
||||
json.field "clen", fmt["contentLength"]
|
||||
json.field "lmt", fmt["lastModified"]
|
||||
json.field "projectionType", fmt["projectionType"]
|
||||
|
||||
fmt_info = itag_to_metadata?(fmt["itag"])
|
||||
if fmt_info
|
||||
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
|
||||
json.field "fps", fps
|
||||
json.field "container", fmt_info["ext"]
|
||||
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
|
||||
|
||||
if fmt_info["height"]?
|
||||
json.field "resolution", "#{fmt_info["height"]}p"
|
||||
|
||||
quality_label = "#{fmt_info["height"]}p"
|
||||
if fps > 30
|
||||
quality_label += "60"
|
||||
end
|
||||
json.field "qualityLabel", quality_label
|
||||
|
||||
if fmt_info["width"]?
|
||||
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "formatStreams" do
|
||||
json.array do
|
||||
self.fmt_stream.each do |fmt|
|
||||
json.object do
|
||||
json.field "url", fmt["url"]
|
||||
json.field "itag", fmt["itag"].as_i.to_s
|
||||
json.field "type", fmt["mimeType"]
|
||||
json.field "quality", fmt["quality"]
|
||||
|
||||
fmt_info = itag_to_metadata?(fmt["itag"])
|
||||
if fmt_info
|
||||
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
|
||||
json.field "fps", fps
|
||||
json.field "container", fmt_info["ext"]
|
||||
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
|
||||
|
||||
if fmt_info["height"]?
|
||||
json.field "resolution", "#{fmt_info["height"]}p"
|
||||
|
||||
quality_label = "#{fmt_info["height"]}p"
|
||||
if fps > 30
|
||||
quality_label += "60"
|
||||
end
|
||||
json.field "qualityLabel", quality_label
|
||||
|
||||
if fmt_info["width"]?
|
||||
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "captions" do
|
||||
json.array do
|
||||
self.captions.each do |caption|
|
||||
json.object do
|
||||
json.field "label", caption.name
|
||||
json.field "languageCode", caption.languageCode
|
||||
json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "recommendedVideos" do
|
||||
json.array do
|
||||
self.related_videos.each do |rv|
|
||||
if rv["id"]?
|
||||
json.object do
|
||||
json.field "videoId", rv["id"]
|
||||
json.field "title", rv["title"]
|
||||
json.field "videoThumbnails" do
|
||||
generate_thumbnails(json, rv["id"])
|
||||
end
|
||||
|
||||
json.field "author", rv["author"]
|
||||
json.field "authorUrl", rv["author_url"]?
|
||||
json.field "authorId", rv["ucid"]?
|
||||
if rv["author_thumbnail"]?
|
||||
json.field "authorThumbnails" do
|
||||
json.array do
|
||||
qualities = {32, 48, 76, 100, 176, 512}
|
||||
|
||||
qualities.each do |quality|
|
||||
json.object do
|
||||
json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-")
|
||||
json.field "width", quality
|
||||
json.field "height", quality
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
|
||||
json.field "viewCountText", rv["short_view_count_text"]?
|
||||
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_json(locale, json : JSON::Builder | Nil = nil)
|
||||
if json
|
||||
to_json(locale, json)
|
||||
else
|
||||
JSON.build do |json|
|
||||
to_json(locale, json)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def title
|
||||
info["videoDetails"]["title"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
def ucid
|
||||
info["videoDetails"]["channelId"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
def author
|
||||
info["videoDetails"]["author"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
def length_seconds : Int32
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
|
||||
info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
|
||||
end
|
||||
|
||||
def views : Int64
|
||||
info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
|
||||
end
|
||||
|
||||
def likes : Int64
|
||||
info["likes"]?.try &.as_i64 || 0_i64
|
||||
end
|
||||
|
||||
def dislikes : Int64
|
||||
info["dislikes"]?.try &.as_i64 || 0_i64
|
||||
end
|
||||
|
||||
def average_rating : Float64
|
||||
# (likes / (likes + dislikes) * 4 + 1)
|
||||
info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
|
||||
end
|
||||
|
||||
def published : Time
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
|
||||
end
|
||||
|
||||
def published=(other : Time)
|
||||
info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
|
||||
end
|
||||
|
||||
def cookie
|
||||
info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
|
||||
end
|
||||
|
||||
def allow_ratings
|
||||
r = info["videoDetails"]["allowRatings"]?.try &.as_bool
|
||||
r.nil? ? false : r
|
||||
end
|
||||
|
||||
def live_now
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]?
|
||||
.try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
|
||||
end
|
||||
|
||||
def is_listed
|
||||
info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
|
||||
end
|
||||
|
||||
def is_upcoming
|
||||
info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
|
||||
end
|
||||
|
||||
def premiere_timestamp : Time?
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]?
|
||||
.try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
|
||||
end
|
||||
|
||||
def keywords
|
||||
info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
|
||||
end
|
||||
|
||||
def related_videos
|
||||
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
|
||||
end
|
||||
|
||||
def allowed_regions
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]?
|
||||
.try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
|
||||
end
|
||||
|
||||
def author_thumbnail : String
|
||||
info["authorThumbnail"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
def sub_count_text : String
|
||||
info["subCountText"]?.try &.as_s || "-"
|
||||
end
|
||||
|
||||
def fmt_stream
|
||||
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
|
||||
|
||||
fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
|
||||
fmt_stream.each do |fmt|
|
||||
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
|
||||
s.each do |k, v|
|
||||
fmt[k] = JSON::Any.new(v)
|
||||
end
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
||||
end
|
||||
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
|
||||
end
|
||||
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
|
||||
@fmt_stream = fmt_stream
|
||||
return @fmt_stream.as(Array(Hash(String, JSON::Any)))
|
||||
end
|
||||
|
||||
def adaptive_fmts
|
||||
return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
|
||||
fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
|
||||
fmt_stream.each do |fmt|
|
||||
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
|
||||
s.each do |k, v|
|
||||
fmt[k] = JSON::Any.new(v)
|
||||
end
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
||||
end
|
||||
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
|
||||
end
|
||||
# See https://github.com/TeamNewPipe/NewPipe/issues/2415
|
||||
# Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
|
||||
fmt_stream.reject! { |f| !f["indexRange"]? }
|
||||
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
|
||||
@adaptive_fmts = fmt_stream
|
||||
return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
|
||||
end
|
||||
|
||||
def video_streams
|
||||
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
|
||||
end
|
||||
|
||||
def audio_streams
|
||||
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
|
||||
end
|
||||
|
||||
def storyboards
|
||||
storyboards = info["storyboards"]?
|
||||
.try &.as_h
|
||||
.try &.["playerStoryboardSpecRenderer"]?
|
||||
.try &.["spec"]?
|
||||
.try &.as_s.split("|")
|
||||
|
||||
if !storyboards
|
||||
if storyboard = info["storyboards"]?
|
||||
.try &.as_h
|
||||
.try &.["playerLiveStoryboardSpecRenderer"]?
|
||||
.try &.["spec"]?
|
||||
.try &.as_s
|
||||
return [{
|
||||
url: storyboard.split("#")[0],
|
||||
width: 106,
|
||||
height: 60,
|
||||
count: -1,
|
||||
interval: 5000,
|
||||
storyboard_width: 3,
|
||||
storyboard_height: 3,
|
||||
storyboard_count: -1,
|
||||
}]
|
||||
end
|
||||
end
|
||||
|
||||
items = [] of NamedTuple(
|
||||
url: String,
|
||||
width: Int32,
|
||||
height: Int32,
|
||||
count: Int32,
|
||||
interval: Int32,
|
||||
storyboard_width: Int32,
|
||||
storyboard_height: Int32,
|
||||
storyboard_count: Int32)
|
||||
|
||||
return items if !storyboards
|
||||
|
||||
url = URI.parse(storyboards.shift)
|
||||
params = HTTP::Params.parse(url.query || "")
|
||||
|
||||
storyboards.each_with_index do |storyboard, i|
|
||||
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#")
|
||||
params["sigh"] = sigh
|
||||
url.query = params.to_s
|
||||
|
||||
width = width.to_i
|
||||
height = height.to_i
|
||||
count = count.to_i
|
||||
interval = interval.to_i
|
||||
storyboard_width = storyboard_width.to_i
|
||||
storyboard_height = storyboard_height.to_i
|
||||
storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
|
||||
|
||||
items << {
|
||||
url: url.to_s.sub("$L", i).sub("$N", "M$M"),
|
||||
width: width,
|
||||
height: height,
|
||||
count: count,
|
||||
interval: interval,
|
||||
storyboard_width: storyboard_width,
|
||||
storyboard_height: storyboard_height,
|
||||
storyboard_count: storyboard_count,
|
||||
}
|
||||
end
|
||||
|
||||
items
|
||||
end
|
||||
|
||||
def paid
|
||||
reason = info["playabilityStatus"]?.try &.["reason"]?
|
||||
paid = reason == "This video requires payment to watch." ? true : false
|
||||
paid
|
||||
end
|
||||
|
||||
def premium
|
||||
keywords.includes? "YouTube Red"
|
||||
end
|
||||
|
||||
def captions : Array(Caption)
|
||||
return @captions.as(Array(Caption)) if @captions
|
||||
captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
|
||||
name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
|
||||
languageCode = caption["languageCode"].to_s
|
||||
baseUrl = caption["baseUrl"].to_s
|
||||
|
||||
caption = Caption.new(name.to_s, languageCode, baseUrl)
|
||||
caption.name = caption.name.split(" - ")[0]
|
||||
caption
|
||||
end
|
||||
captions ||= [] of Caption
|
||||
@captions = captions
|
||||
return @captions.as(Array(Caption))
|
||||
end
|
||||
|
||||
def description
|
||||
description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
|
||||
.try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
# TODO
|
||||
def description=(value : String)
|
||||
@description = value
|
||||
end
|
||||
|
||||
def description_html
|
||||
info["descriptionHtml"]?.try &.as_s || "<p></p>"
|
||||
end
|
||||
|
||||
def description_html=(value : String)
|
||||
info["descriptionHtml"] = JSON::Any.new(value)
|
||||
end
|
||||
|
||||
def short_description
|
||||
info["shortDescription"]?.try &.as_s? || ""
|
||||
end
|
||||
|
||||
def hls_manifest_url : String?
|
||||
info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s
|
||||
end
|
||||
|
||||
def dash_manifest_url
|
||||
info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
|
||||
end
|
||||
|
||||
def genre : String
|
||||
info["genre"]?.try &.as_s || ""
|
||||
end
|
||||
|
||||
def genre_url : String?
|
||||
info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
|
||||
end
|
||||
|
||||
def license : String?
|
||||
info["license"]?.try &.as_s
|
||||
end
|
||||
|
||||
def is_family_friendly : Bool
|
||||
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
|
||||
end
|
||||
|
||||
def is_vr : Bool
|
||||
info["streamingData"]?.try &.["adaptiveFormats"].as_a[0]?.try &.["projectionType"].as_s == "MESH" ? true : false || false
|
||||
end
|
||||
|
||||
def wilson_score : Float64
|
||||
ci_lower_bound(likes, likes + dislikes).round(4)
|
||||
end
|
||||
|
||||
def engagement : Float64
|
||||
(((likes + dislikes) / views) * 100).round(4)
|
||||
end
|
||||
|
||||
def reason : String?
|
||||
info["reason"]?.try &.as_s
|
||||
end
|
||||
|
||||
def session_token : String?
|
||||
info["sessionToken"]?.try &.as_s?
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -99,7 +99,7 @@ private module Parsers
|
|||
end
|
||||
end
|
||||
|
||||
SearchVideo.new({
|
||||
YouTubeStructs::VideoRenderer.new({
|
||||
title: title,
|
||||
id: video_id,
|
||||
author: author,
|
||||
|
|
@ -149,7 +149,7 @@ private module Parsers
|
|||
video_count = HelperExtractors.get_video_count(item_contents)
|
||||
description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
|
||||
|
||||
SearchChannel.new({
|
||||
YouTubeStructs::ChannelRenderer.new({
|
||||
author: author,
|
||||
ucid: author_id,
|
||||
author_thumbnail: author_thumbnail,
|
||||
|
|
@ -184,13 +184,13 @@ private module Parsers
|
|||
video_count = HelperExtractors.get_video_count(item_contents)
|
||||
playlist_thumbnail = HelperExtractors.get_thumbnails(item_contents)
|
||||
|
||||
SearchPlaylist.new({
|
||||
YouTubeStructs::PlaylistRenderer.new({
|
||||
title: title,
|
||||
id: plid,
|
||||
author: author_fallback.name,
|
||||
ucid: author_fallback.id,
|
||||
video_count: video_count,
|
||||
videos: [] of SearchPlaylistVideo,
|
||||
videos: [] of YouTubeStructs::PlaylistVideoRenderer,
|
||||
thumbnail: playlist_thumbnail,
|
||||
})
|
||||
end
|
||||
|
|
@ -227,16 +227,16 @@ private module Parsers
|
|||
v_title = v.dig?("title", "simpleText").try &.as_s || ""
|
||||
v_id = v["videoId"]?.try &.as_s || ""
|
||||
v_length_seconds = v.dig?("lengthText", "simpleText").try { |t| decode_length_seconds(t.as_s) } || 0
|
||||
SearchPlaylistVideo.new({
|
||||
title: v_title,
|
||||
id: v_id,
|
||||
YouTubeStructs::PlaylistVideoRenderer.new(
|
||||
title: v_title,
|
||||
id: v_id,
|
||||
length_seconds: v_length_seconds,
|
||||
})
|
||||
end || [] of SearchPlaylistVideo
|
||||
)
|
||||
end || [] of YouTubeStructs::PlaylistVideoRenderer
|
||||
|
||||
# TODO: item_contents["publishedTimeText"]?
|
||||
|
||||
SearchPlaylist.new({
|
||||
YouTubeStructs::PlaylistRenderer.new({
|
||||
title: title,
|
||||
id: plid,
|
||||
author: author,
|
||||
|
|
@ -281,7 +281,7 @@ private module Parsers
|
|||
description_html = item_contents["subtitle"]?.try { |desc| parse_content(desc) } || ""
|
||||
|
||||
# Content parsing
|
||||
contents = [] of SearchItem
|
||||
contents = [] of YouTubeStructs::Renderer
|
||||
|
||||
# Content could be in three locations.
|
||||
if content_container = item_contents["content"]["horizontalListRenderer"]?
|
||||
|
|
@ -299,7 +299,7 @@ private module Parsers
|
|||
end
|
||||
end
|
||||
|
||||
Category.new({
|
||||
YouTubeStructs::Category.new({
|
||||
title: title,
|
||||
contents: contents,
|
||||
description_html: description_html,
|
||||
|
|
@ -538,8 +538,8 @@ end
|
|||
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
|
||||
# The end result is an array of SearchItem.
|
||||
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
|
||||
author_id_fallback : String? = nil) : Array(SearchItem)
|
||||
items = [] of SearchItem
|
||||
author_id_fallback : String? = nil) : Array(YouTubeStructs::Renderer)
|
||||
items = [] of YouTubeStructs::Renderer
|
||||
|
||||
if unpackaged_data = initial_data["contents"]?.try &.as_h
|
||||
elsif unpackaged_data = initial_data["response"]?.try &.as_h
|
||||
|
|
@ -564,3 +564,32 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
|
|||
|
||||
return items
|
||||
end
|
||||
|
||||
# Flattens all items from extracted items into a one dimensional array
|
||||
def flatten_items(items, target = nil)
|
||||
if target.nil?
|
||||
target = [] of YouTubeStructs::Renderer
|
||||
end
|
||||
|
||||
items.each do |i|
|
||||
if i.is_a?(YouTubeStructs::Category)
|
||||
target = target += i.extract_renderers
|
||||
else
|
||||
target << i
|
||||
end
|
||||
end
|
||||
|
||||
return target
|
||||
end
|
||||
|
||||
# Extracts videos (videoRenderer) from initial InnerTube response.
|
||||
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
|
||||
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
|
||||
target = flatten_items(extracted)
|
||||
return target.select(&.is_a?(YouTubeStructs::VideoRenderer)).map(&.as(YouTubeStructs::VideoRenderer))
|
||||
end
|
||||
|
||||
# Extract the selected tab from the array of tabs YouTube returns
|
||||
def extract_selected_tab(tabs)
|
||||
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"]
|
||||
end
|
||||
|
|
|
|||
|
|
@ -247,25 +247,6 @@ def html_to_content(description_html : String)
|
|||
return description
|
||||
end
|
||||
|
||||
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
|
||||
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
|
||||
|
||||
target = [] of SearchItem
|
||||
extracted.each do |i|
|
||||
if i.is_a?(Category)
|
||||
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
|
||||
else
|
||||
target << i
|
||||
end
|
||||
end
|
||||
return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
|
||||
end
|
||||
|
||||
def extract_selected_tab(tabs)
|
||||
# Extract the selected tab from the array of tabs Youtube returns
|
||||
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"]
|
||||
end
|
||||
|
||||
def fetch_continuation_token(items : Array(JSON::Any))
|
||||
# Fetches the continuation token from an array of items
|
||||
return items.last["continuationItemRenderer"]?
|
||||
|
|
|
|||
Loading…
Reference in a new issue