From 3c01bbb0b324233cdd02322db3c3181c59cff904 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 6 Aug 2021 00:31:06 -0700 Subject: [PATCH] Initial extraction for complete struct overhaul --- .../data_structs/channel-information.cr | 36 +++++ src/invidious/data_structs/channel.cr | 115 ++++++++++++++++ src/invidious/data_structs/renderers/any.cr | 57 ++++++++ .../data_structs/renderers/category.cr | 38 ++++++ .../renderers/channel_renderer.cr | 62 +++++++++ .../renderers/playlist_renderer.cr | 64 +++++++++ .../data_structs/renderers/video_renderer.cr | 128 ++++++++++++++++++ 7 files changed, 500 insertions(+) create mode 100644 src/invidious/data_structs/channel-information.cr create mode 100644 src/invidious/data_structs/channel.cr create mode 100644 src/invidious/data_structs/renderers/any.cr create mode 100644 src/invidious/data_structs/renderers/category.cr create mode 100644 src/invidious/data_structs/renderers/channel_renderer.cr create mode 100644 src/invidious/data_structs/renderers/playlist_renderer.cr create mode 100644 src/invidious/data_structs/renderers/video_renderer.cr diff --git a/src/invidious/data_structs/channel-information.cr b/src/invidious/data_structs/channel-information.cr new file mode 100644 index 00000000..d4f7ecea --- /dev/null +++ b/src/invidious/data_structs/channel-information.cr @@ -0,0 +1,36 @@ +module YTStructs + # Struct to represent channel heading information. + # + # As of master this is mostly taken from the about tab. + # + # TODO: Refactor into into ChannelInformation + struct AboutChannel + include DB::Serializable + + property ucid : String + property author : String + property auto_generated : Bool + property author_url : String + property author_thumbnail : String + property banner : String? + property description_html : String + property paid : Bool + property total_views : Int64 + property sub_count : Int32 + property joined : Time + property is_family_friendly : Bool + property allowed_regions : Array(String) + property related_channels : Array(AboutRelatedChannel) + property tabs : Array(String) + end + + # TODO this should be removed. YouTube has removed related channels. + struct AboutRelatedChannel + include DB::Serializable + + property ucid : String + property author : String + property author_url : String + property author_thumbnail : String + end +end diff --git a/src/invidious/data_structs/channel.cr b/src/invidious/data_structs/channel.cr new file mode 100644 index 00000000..0b0785db --- /dev/null +++ b/src/invidious/data_structs/channel.cr @@ -0,0 +1,115 @@ +module InvidiousStructs + # Struct for representing a cached Invidious channel. + # + # Currently used for storing subscriptions. + struct InvidiousChannel + include DB::Serializable + + property id : String + property author : String + property updated : Time + property deleted : Bool + property subscribed : Time? + end +end + +module YTStructs + struct ChannelVideo + include DB::Serializable + + property id : String + property title : String + property published : Time + property updated : Time + property ucid : String + property author : String + property length_seconds : Int32 = 0 + property live_now : Bool = false + property premiere_timestamp : Time? = nil + property views : Int64? = nil + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "shortVideo" + + json.field "title", self.title + json.field "videoId", self.id + json.field "videoThumbnails" do + generate_thumbnails(json, self.id) + end + + json.field "lengthSeconds", self.length_seconds + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + json.field "published", self.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) + + json.field "viewCount", self.views + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + + def to_xml(locale, query_params, xml : XML::Builder) + query_params["v"] = self.id + + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") + + xml.element("author") do + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + end + end + end + + def to_xml(locale, xml : XML::Builder | Nil = nil) + if xml + to_xml(locale, xml) + else + XML.build do |xml| + to_xml(locale, xml) + end + end + end + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map { |var| var.name }}} + } + {% end %} + end + end +end diff --git a/src/invidious/data_structs/renderers/any.cr b/src/invidious/data_structs/renderers/any.cr new file mode 100644 index 00000000..13cdbd8a --- /dev/null +++ b/src/invidious/data_structs/renderers/any.cr @@ -0,0 +1,57 @@ +module YTStructs + alias Renderers = VideoRenderer | ChannelRenderer | PlaylistRenderer | Category + + # Wrapper object around all renderer types + struct AnyRenderer + def initialize(@raw : Renderers) + end + + # Checks that the underlying value is `VideoRenderer`, and returns its value. + # Raises otherwise + def as_video + @raw.as(VideoRenderer) + end + + # Checks that the underlying value is `VideoRenderer`, and returns its value. + # Returns `nil` otherwise + def as_video? + as_video if @raw.is_a? VideoRenderer + end + + # Checks that the underlying value is `ChannelRenderer`, and returns its value. + # Raises otherwise + def as_channel + @raw.as(ChannelRenderer) + end + + # Checks that the underlying value is `ChannelRenderer`, and returns its value. + # Raises otherwise + def as_channel? + as_channel if @raw.is_a? ChannelRenderer + end + + # Checks that the underlying value is `PlaylistRenderer`, and returns its value. + # Raises otherwise + def as_playlist + @raw.as(PlaylistRenderer) + end + + # Checks that the underlying value is `PlaylistRenderer`, and returns its value. + # Raises otherwise + def as_playlist? + as_playlist if @raw.is_a? PlaylistRenderer + end + + # Checks that the underlying value is `Category`, and returns its value. + # Raises otherwise + def as_category + @raw.as(Category) + end + + # Checks that the underlying value is `Category`, and returns its value. + # Raises otherwise + def as_category? + as_category if @raw.is_a? Category + end + end +end diff --git a/src/invidious/data_structs/renderers/category.cr b/src/invidious/data_structs/renderers/category.cr new file mode 100644 index 00000000..31d3fe1a --- /dev/null +++ b/src/invidious/data_structs/renderers/category.cr @@ -0,0 +1,38 @@ +module YTStructs + # Struct to represent an InnerTube `"shelfRenderers"` + # + # A shelfRenderer renders divided sections on YouTube. IE "People also watched" in search results and + # the various organizational sections in the channel home page. A separate one (richShelfRenderer) is used + # for YouTube home. A shelfRenderer can also sometimes be expanded to show more content within it. + # + # See specs for example JSON response + # + # `shelfRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + class Category + include DB::Serializable + + property title : String + property contents : Array(SearchItem) | Array(Video) + property url : String? + property description_html : String + property badges : Array(Tuple(String, String))? + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "title", self.title + json.field "contents", self.contents + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + end +end diff --git a/src/invidious/data_structs/renderers/channel_renderer.cr b/src/invidious/data_structs/renderers/channel_renderer.cr new file mode 100644 index 00000000..ca522ec6 --- /dev/null +++ b/src/invidious/data_structs/renderers/channel_renderer.cr @@ -0,0 +1,62 @@ +module YTStructs + # Struct to represent an InnerTube `"channelRenderer"` + # + # A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not** + # the channel page itself. + # + # See specs for example JSON response + # + # `channelRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + struct ChannelRenderer + include DB::Serializable + + property author : String + property ucid : String + property author_thumbnail : String + property subscriber_count : Int32 + property video_count : Int32 + property description_html : String + property auto_generated : Bool + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "channel" + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "autoGenerated", self.auto_generated + json.field "subCount", self.subscriber_count + json.field "videoCount", self.video_count + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + end +end diff --git a/src/invidious/data_structs/renderers/playlist_renderer.cr b/src/invidious/data_structs/renderers/playlist_renderer.cr new file mode 100644 index 00000000..2f63da7c --- /dev/null +++ b/src/invidious/data_structs/renderers/playlist_renderer.cr @@ -0,0 +1,64 @@ +module YTStructs + alias PlaylistRendererVideo = NamedTuple(title: String, id: String, length_seconds: Int32) + + # Struct to represent an InnerTube `"PlaylistRenderer"` + # + # A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI. + # It is **not** the playlist itself. + # + # See specs for example JSON response + # + # `PlaylistRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + struct PlaylistRenderer + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property video_count : Int32 + property videos : Array(PlaylistRendererVideo) + property thumbnail : String? + + def to_json(locale, json : JSON::Builder) + json.object do + json.field "type", "playlist" + json.field "title", self.title + json.field "playlistId", self.id + json.field "playlistThumbnail", self.thumbnail + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoCount", self.video_count + json.field "videos" do + json.array do + self.videos.each do |video| + json.object do + json.field "title", video.title + json.field "videoId", video.id + json.field "lengthSeconds", video.length_seconds + + json.field "videoThumbnails" do + generate_thumbnails(json, video.id) + end + end + end + end + end + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + end +end diff --git a/src/invidious/data_structs/renderers/video_renderer.cr b/src/invidious/data_structs/renderers/video_renderer.cr new file mode 100644 index 00000000..bdc24437 --- /dev/null +++ b/src/invidious/data_structs/renderers/video_renderer.cr @@ -0,0 +1,128 @@ +module YTStructs + # Struct to represent an InnerTube `"videoRenderer"` + # + # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** + # the watchable video itself. + # + # See specs for example JSON response + # + # `videoRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. + # + struct VideoRenderer + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property published : Time + property views : Int64 + property description_html : String + property length_seconds : Int32 + property live_now : Bool + property paid : Bool + property premium : Bool + property premiere_timestamp : Time? + + def to_xml(auto_generated, query_params, xml : XML::Builder) + query_params["v"] = self.id + + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") + + xml.element("author") do + if auto_generated + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + else + xml.element("name") { xml.text author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } + end + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + + xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + xml.element("media:description") { xml.text html_to_content(self.description_html) } + end + + xml.element("media:community") do + xml.element("media:statistics", views: self.views) + end + end + end + + def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) + if xml + to_xml(HOST_URL, auto_generated, query_params, xml) + else + XML.build do |json| + to_xml(HOST_URL, auto_generated, query_params, xml) + end + end + end + + def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) + json.object do + json.field "type", "video" + json.field "title", self.title + json.field "videoId", self.id + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoThumbnails" do + generate_thumbnails(json, self.id) + end + + json.field "description", html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + + json.field "viewCount", self.views + json.field "published", self.published.to_unix + json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) + json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now + json.field "paid", self.paid + json.field "premium", self.premium + json.field "isUpcoming", self.is_upcoming + + if self.premiere_timestamp + json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix + end + end + end + + def to_json(locale, json : JSON::Builder | Nil = nil) + if json + to_json(locale, json) + else + JSON.build do |json| + to_json(locale, json) + end + end + end + + def is_upcoming + premiere_timestamp ? true : false + end + end +end