Add parser for categories (shelfRenderer)

This commit adds a new parser for YT's shelfRenderers which are
typically used to denote different categories.The code for featured
channels parsing has also been moved to use the new parser but some
additional refactoring are needed there.

The ContinuationExtractor has also been improved and is now capable of
extraction continuation data that is packaged under
"appendContinuationItemsAction"

In additional this commit adds some useful helper functions to extract
the current selected tab the continuation token. This is to mainly
reduce code size and repetition.

src/invidious/channels.cr

@@ -380,24 +380,73 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
   return items, continuation
 end
 
-def fetch_channel_featured_channels(ucid, tab_data, params = nil, continuation = nil, title = nil)
+def fetch_channel_featured_channels(ucid, tab_data, params = nil, continuation = nil, query_title = nil)
   if continuation.is_a?(String)
     initial_data = request_youtube_api_browse(continuation)
-    channels_tab_content = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
+    items = extract_items(initial_data)
+    continuation_token = fetch_continuation_token(initial_data)
 
-    return process_featured_channels([channels_tab_content], nil, title, continuation_items = true)
+    return [Category.new({
+      title:                query_title.not_nil!, # If continuation contents is requested then the query_title has to be passed along.
+      contents:             items,
+      browse_endpoint_data: nil,
+      continuation_token:   continuation_token,
+      badges:               nil,
+    })]
   else
     if params.is_a?(String)
       initial_data = request_youtube_api_browse(ucid, params)
+      continuation_token = fetch_continuation_token(initial_data)
     else
       initial_data = request_youtube_api_browse(ucid, tab_data[1])
+      continuation_token = nil
     end
 
-    channels_tab = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][tab_data[0]]["tabRenderer"]
+    channels_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
-    channels_tab_content = channels_tab["content"]["sectionListRenderer"]["contents"].as_a
+    submenu = channels_tab["content"]["sectionListRenderer"]["subMenu"]?
-    submenu_data = channels_tab["content"]["sectionListRenderer"]["subMenu"]?.try &.["channelSubMenuRenderer"]["contentTypeSubMenuItems"] || false
 
-    return process_featured_channels(channels_tab_content, submenu_data)
+    # There's no submenu data if the channel doesn't feature any channels.
+    if !submenu
+      return [] of Category
+    end
+
+    submenu_data = submenu["channelSubMenuRenderer"]["contentTypeSubMenuItems"]
+      
+    items = extract_items(initial_data)
+    fallback_title = submenu_data.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
+
+    # Although extract_items parsed everything into the right structs, we still have
+    # to fill in the title (if missing) attribute since Youtube doesn't return it when requesting
+    # a full category
+
+    category_array = [] of Category
+    items.each do |category|
+      # Tell compiler that the result from extract_items has to be an array of Categories
+      if !category.is_a?(Category)
+        next
+      end
+
+      category_array << Category.new({
+        title:                category.title.empty? ? fallback_title : category.title,
+        contents:             category.contents,
+        browse_endpoint_data: category.browse_endpoint_data,
+        continuation_token:   continuation_token,
+        badges:               nil,
+      })
+    end
+
+    # If we don't have any categories we'll create one.
+    if category_array.empty?
+      return [Category.new({
+        title:                fallback_title, # If continuation contents is requested then the query_title has to be passed along.
+        contents:             items,
+        browse_endpoint_data: nil,
+        continuation_token:   continuation_token,
+        badges:               nil,
+      })]
+    end
+
+    return category_array
   end
 end
 

src/invidious/featured_channels.cr

@@ -1,170 +0,0 @@
-struct FeaturedChannel
-  include DB::Serializable
-
-  property author : String
-  property ucid : String
-  property author_thumbnail : String
-  property subscriber_count : Int32
-  property video_count : Int32
-  property description_html : String?
-
-  def to_json(locale, json : JSON::Builder)
-    json.object do
-      json.field "author", self.author
-      json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
-      json.field "authorThumbnails" do
-        json.array do
-          qualities = {32, 48, 76, 100, 176, 512}
-
-          qualities.each do |quality|
-            json.object do
-              json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
-              json.field "width", quality
-              json.field "height", quality
-            end
-          end
-        end
-      end
-
-      json.field "description", html_to_content(self.description_html)
-      json.field "descriptionHtml", self.description_html
-      json.field "subCount", self.subscriber_count
-      json.field "videoCount", self.video_count
-      json.field "badges", self.badges
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder | Nil = nil)
-    if json
-      to_json(locale, json)
-    else
-      JSON.build do |json|
-        to_json(locale, json)
-      end
-    end
-  end
-end
-
-struct Category
-  include DB::Serializable
-
-  property title : String
-  property contents : Array(FeaturedChannel) | FeaturedChannel
-  property browse_endpoint_param : String?
-  property continuation_token : String?
-
-  def to_json(locale, json : JSON::Builder)
-    json.object do
-      json.field "title", self.title
-      json.field "contents", self.contents
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder | Nil = nil)
-    if json
-      to_json(locale, json)
-    else
-      JSON.build do |json|
-        to_json(locale, json)
-      end
-    end
-  end
-end
-
-def _extract_channel_data(channel)
-  ucid = channel["channelId"].as_s
-  author = channel["title"]["simpleText"].as_s
-  author_thumbnail = channel["thumbnail"]["thumbnails"].as_a[0]["url"].as_s
-  subscriber_count = channel["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
-    .try { |text| short_text_to_number(text.split(" ")[0]) } || 0
-
-  video_count = channel["videoCountText"]?.try &.["runs"][0]["text"].as_s.gsub(/\D/, "").to_i || 0
-
-  if channel["descriptionSnippet"]?
-    description = channel["descriptionSnippet"]["runs"][0]["text"].as_s
-    description_html = HTML.escape(description).gsub("\n", "")
-  else
-    description_html = nil
-  end
-
-  FeaturedChannel.new({
-    author:           author,
-    ucid:             ucid,
-    author_thumbnail: author_thumbnail,
-    subscriber_count: subscriber_count,
-    video_count:      video_count,
-    description_html: description_html,
-  })
-end
-
-def process_featured_channels(data, submenu_data, title = nil, continuation_items = false)
-  all_categories = [] of Category
-
-  if submenu_data.is_a?(Bool)
-    return all_categories
-  end
-
-  # Extraction process differs when there's more than one category
-  if data.size > 1
-    data.each do |raw_category|
-      raw_category = raw_category["itemSectionRenderer"]["contents"].as_a[0]["shelfRenderer"]
-
-      category_title = raw_category["title"]["runs"][0]["text"].as_s
-      browse_endpoint_param = raw_category["endpoint"]["browseEndpoint"]["params"].as_s
-
-      # Category has multiple channels
-      if raw_category["content"].as_h.has_key?("horizontalListRenderer")
-        contents = [] of FeaturedChannel
-        raw_category["content"]["horizontalListRenderer"]["items"].as_a.each do |channel|
-          contents << _extract_channel_data(channel["gridChannelRenderer"])
-        end
-        # Single channel
-      else
-        channel = raw_category["content"]["expandedShelfContentsRenderer"]["items"][0]["channelRenderer"]
-        contents = _extract_channel_data(channel)
-      end
-
-      all_categories << Category.new({
-        title:                 category_title,
-        contents:              contents,
-        browse_endpoint_param: browse_endpoint_param,
-        continuation_token:    nil,
-      })
-    end
-  else
-    if !continuation_items
-      raw_category_contents = data[0]["itemSectionRenderer"]["contents"].as_a[0]["gridRenderer"]["items"].as_a
-    else
-      raw_category_contents = data[0].as_a
-    end
-
-    category_title = submenu_data.try &.[0]["title"].as_s || title || ""
-
-    browse_endpoint_param = nil # Not needed
-    continuation_token = nil
-
-    # If a continuation token is needed it'll always be after at least twelve channels
-    if raw_category_contents.size > 12
-      continuation_token = raw_category_contents[-1]["continuationItemRenderer"]?.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s || nil
-
-      if !continuation_token.nil?
-        raw_category_contents = raw_category_contents[0..-2]
-      end
-    end
-
-    contents = [] of FeaturedChannel
-    raw_category_contents.each do |channel|
-      contents << _extract_channel_data(channel["gridChannelRenderer"])
-    end
-
-    all_categories << Category.new({
-      title:                 category_title,
-      contents:              contents,
-      browse_endpoint_param: browse_endpoint_param,
-      continuation_token:    continuation_token,
-    })
-  end
-
-  return all_categories
-end

src/invidious/helpers/extractors.cr

@@ -5,7 +5,7 @@
 private ITEM_CONTAINER_EXTRACTOR = {
   YoutubeTabsExtractor.new,
   SearchResultsExtractor.new,
-  ContinuationExtractor.new
+  ContinuationExtractor.new,
 }
 
 private ITEM_PARSERS = {
@@ -13,6 +13,7 @@ private ITEM_PARSERS = {
   ChannelParser.new,
   GridPlaylistParser.new,
   PlaylistParser.new,
+  CategoryParser.new,
 }
 
 private struct AuthorFallback
@@ -98,7 +99,7 @@ end
 
 private class ChannelParser < ItemParser
   def process(item, author_fallback)
-    if item_contents = item["channelRenderer"]?
+    if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
       return self.parse(item_contents, author_fallback)
     end
   end
@@ -197,6 +198,88 @@ private class PlaylistParser < ItemParser
   end
 end
 
+private class CategoryParser < ItemParser
+  def process(item, author_fallback)
+    if item_contents = item["shelfRenderer"]?
+      return self.parse(item_contents, author_fallback)
+    end
+  end
+
+  def parse(item_contents, author_fallback)
+    # Title extraction is a bit complicated. There are two possible routes for it
+    # as well as times when the title attribute just isn't sent by YT.
+
+    title_container = item_contents["title"]? || ""
+    if !title_container.is_a? String
+      if title = title_container["simpleText"]?
+        title = title.as_s
+      else
+        title = title_container["runs"][0]["text"].as_s
+      end
+    else
+      title = ""
+    end
+
+    browse_endpoint = item_contents["endpoint"]?.try &.["browseEndpoint"] || nil
+    browse_endpoint_data = ""
+    category_type = 0 # 0: Video, 1: Channels, 2: Playlist/feed, 3: trending
+
+    # There's no endpoint data for video and trending category
+    if !item_contents["endpoint"]?
+      if !item_contents["videoId"]?
+        category_type = 3
+      end
+    end
+
+    if !browse_endpoint.nil?
+      # Playlist/feed categories doesn't need the params value (nor is it even included in yt response)
+      # instead it uses the browseId parameter. So if there isn't a params value we can assume the
+      # category is a playlist/feed
+      if browse_endpoint["params"]?
+        browse_endpoint_data = browse_endpoint["params"].as_s
+        category_type = 1
+      else
+        browse_endpoint_data = browse_endpoint["browseId"].as_s
+        category_type = 2
+      end
+    end
+
+    # Sometimes a category can have badges.
+    badges = [] of Tuple(String, String) # (Badge style, label)
+    item_contents["badges"]?.try &.as_a.each do |badge|
+      badge = badge["metadataBadgeRenderer"]
+      badges << {badge["style"].as_s, badge["label"].as_s}
+    end
+
+    # Content parsing
+    contents = [] of SearchItem
+
+    # Content could be in three locations.
+    if content_container = item_contents["content"]["horizontalListRenderer"]?
+    elsif content_container = item_contents["content"]["expandedShelfContentsRenderer"]
+    elsif content_container = item_contents["content"]["verticalListRenderer"]
+    else
+      content_container = item_contents["contents"]
+    end
+
+    raw_contents = content_container["items"].as_a
+    raw_contents.each do |item|
+      result = extract_item(item)
+      if !result.nil?
+        contents << result
+      end
+    end
+
+    Category.new({
+      title:                title,
+      contents:             contents,
+      browse_endpoint_data: browse_endpoint_data,
+      continuation_token:   nil,
+      badges:               badges,
+    })
+  end
+end
+
 # The following are the extractors for extracting an array of items from
 # the internal Youtube API's JSON response. The result is then packaged into
 # a structure we can more easily use via the parsers above. Their internals are
@@ -220,25 +303,22 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
   private def extract(target)
     raw_items = [] of JSON::Any
     selected_tab = extract_selected_tab(target["tabs"])
-    content = selected_tab["tabRenderer"]["content"]
+    content = selected_tab["content"]
 
-    content["sectionListRenderer"]["contents"].as_a.each do | renderer_container |
+    content["sectionListRenderer"]["contents"].as_a.each do |renderer_container|
       renderer_container = renderer_container["itemSectionRenderer"]
       renderer_container_contents = renderer_container["contents"].as_a[0]
 
-      # Shelf renderer usually refer to a category and would need special handling once
+      # Category extraction
-      # An extractor for categories are added. But for now it is just used to 
-      # extract items for the trending page
       if items_container = renderer_container_contents["shelfRenderer"]?
-        if items_container["content"]["expandedShelfContentsRenderer"]?
+        raw_items << renderer_container_contents
-           items_container = items_container["content"]["expandedShelfContentsRenderer"]
+        next
-        end
       elsif items_container = renderer_container_contents["gridRenderer"]?
       else
         items_container = renderer_container_contents
       end
 
-      items_container["items"].as_a.each do | item |
+      items_container["items"].as_a.each do |item|
         raw_items << item
       end
     end
@@ -268,6 +348,8 @@ private class ContinuationExtractor < ItemsContainerExtractor
   def process(initial_data)
     if target = initial_data["continuationContents"]?
       self.extract(target)
+    elsif target = initial_data["appendContinuationItemsAction"]?
+      self.extract(target)
     end
   end
 
@@ -275,13 +357,16 @@ private class ContinuationExtractor < ItemsContainerExtractor
     raw_items = [] of JSON::Any
     if content = target["gridContinuation"]?
       raw_items = content["items"].as_a
+    elsif content = target["continuationItems"]?
+      raw_items = content.as_a
     end
 
     return raw_items
   end
 end
 
-def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil)
+def extract_item(item : JSON::Any, author_fallback : String? = nil,
+                 author_id_fallback : String? = nil)
   # Parses an item from Youtube's JSON response into a more usable structure.
   # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
   author_fallback = AuthorFallback.new(author_fallback, author_id_fallback)
@@ -298,21 +383,29 @@ def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fa
   # TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer
 end
 
-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
+def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
+                  author_id_fallback : String? = nil)
   items = [] of SearchItem
-  initial_data = initial_data["contents"]?.try &.as_h  || initial_data["response"]?.try &.as_h || initial_data
+
+  if unpackaged_data = initial_data["contents"]?.try &.as_h
+  elsif unpackaged_data = initial_data["response"]?.try &.as_h
+  elsif unpackaged_data = initial_data["onResponseReceivedActions"]?.try &.as_a.[0].as_h
+  else
+    unpackaged_data = initial_data
+  end
 
   # This is identicial to the parser cyling of extract_item().
-  ITEM_CONTAINER_EXTRACTOR.each do | extractor |
+  ITEM_CONTAINER_EXTRACTOR.each do |extractor|
-    results = extractor.process(initial_data)
+    results = extractor.process(unpackaged_data)
     if !results.nil?
-      results.each do | item |
+      results.each do |item|
         parsed_result = extract_item(item, author_fallback, author_id_fallback)
 
         if !parsed_result.nil?
           items << parsed_result
         end
       end
+      return items      
     end
   end
 

src/invidious/helpers/helpers.cr

@@ -248,12 +248,38 @@ def html_to_content(description_html : String)
 end
 
 def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
-  extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
+  extracted = extract_items(initial_data, author_fallback, author_id_fallback)
+
+  if extracted.is_a?(Category)
+    target = extracted.contents
+  else
+    target = extracted
+  end
+  return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
 end
 
 def extract_selected_tab(tabs)
   # Extract the selected tab from the array of tabs Youtube returns
-  return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]
+  return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"]
+end
+
+def fetch_continuation_token(items : Array(JSON::Any))
+  # Fetches the continuation token from an array of items
+  return items.last["continuationItemRenderer"]?
+  .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
+
+end
+
+def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
+  # Fetches the continuation token from initial data
+  if initial_data["onResponseReceivedActions"]?
+    continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
+  else
+    tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
+    continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
+  end
+
+  return fetch_continuation_token(continuation_items.as_a)
 end
 
 def check_enum(db, enum_name, struct_type = nil)

src/invidious/helpers/invidiousitems.cr

@@ -0,0 +1,258 @@
+struct SearchVideo
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property ucid : String
+  property published : Time
+  property views : Int64
+  property description_html : String
+  property length_seconds : Int32
+  property live_now : Bool
+  property paid : Bool
+  property premium : Bool
+  property premiere_timestamp : Time?
+
+  def to_xml(auto_generated, query_params, xml : XML::Builder)
+    query_params["v"] = self.id
+
+    xml.element("entry") do
+      xml.element("id") { xml.text "yt:video:#{self.id}" }
+      xml.element("yt:videoId") { xml.text self.id }
+      xml.element("yt:channelId") { xml.text self.ucid }
+      xml.element("title") { xml.text self.title }
+      xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
+
+      xml.element("author") do
+        if auto_generated
+          xml.element("name") { xml.text self.author }
+          xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
+        else
+          xml.element("name") { xml.text author }
+          xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+        end
+      end
+
+      xml.element("content", type: "xhtml") do
+        xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
+          xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
+            xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
+          end
+
+          xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
+        end
+      end
+
+      xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
+
+      xml.element("media:group") do
+        xml.element("media:title") { xml.text self.title }
+        xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
+          width: "320", height: "180")
+        xml.element("media:description") { xml.text html_to_content(self.description_html) }
+      end
+
+      xml.element("media:community") do
+        xml.element("media:statistics", views: self.views)
+      end
+    end
+  end
+
+  def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
+    if xml
+      to_xml(HOST_URL, auto_generated, query_params, xml)
+    else
+      XML.build do |json|
+        to_xml(HOST_URL, auto_generated, query_params, xml)
+      end
+    end
+  end
+
+  def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
+    json.object do
+      json.field "type", "video"
+      json.field "title", self.title
+      json.field "videoId", self.id
+
+      json.field "author", self.author
+      json.field "authorId", self.ucid
+      json.field "authorUrl", "/channel/#{self.ucid}"
+
+      json.field "videoThumbnails" do
+        generate_thumbnails(json, self.id)
+      end
+
+      json.field "description", html_to_content(self.description_html)
+      json.field "descriptionHtml", self.description_html
+
+      json.field "viewCount", self.views
+      json.field "published", self.published.to_unix
+      json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
+      json.field "lengthSeconds", self.length_seconds
+      json.field "liveNow", self.live_now
+      json.field "paid", self.paid
+      json.field "premium", self.premium
+      json.field "isUpcoming", self.is_upcoming
+
+      if self.premiere_timestamp
+        json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
+      end
+    end
+  end
+
+  def to_json(locale, json : JSON::Builder | Nil = nil)
+    if json
+      to_json(locale, json)
+    else
+      JSON.build do |json|
+        to_json(locale, json)
+      end
+    end
+  end
+
+  def is_upcoming
+    premiere_timestamp ? true : false
+  end
+end
+
+struct SearchPlaylistVideo
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property length_seconds : Int32
+end
+
+struct SearchPlaylist
+  include DB::Serializable
+
+  property title : String
+  property id : String
+  property author : String
+  property ucid : String
+  property video_count : Int32
+  property videos : Array(SearchPlaylistVideo)
+  property thumbnail : String?
+
+  def to_json(locale, json : JSON::Builder)
+    json.object do
+      json.field "type", "playlist"
+      json.field "title", self.title
+      json.field "playlistId", self.id
+      json.field "playlistThumbnail", self.thumbnail
+
+      json.field "author", self.author
+      json.field "authorId", self.ucid
+      json.field "authorUrl", "/channel/#{self.ucid}"
+
+      json.field "videoCount", self.video_count
+      json.field "videos" do
+        json.array do
+          self.videos.each do |video|
+            json.object do
+              json.field "title", video.title
+              json.field "videoId", video.id
+              json.field "lengthSeconds", video.length_seconds
+
+              json.field "videoThumbnails" do
+                generate_thumbnails(json, video.id)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def to_json(locale, json : JSON::Builder | Nil = nil)
+    if json
+      to_json(locale, json)
+    else
+      JSON.build do |json|
+        to_json(locale, json)
+      end
+    end
+  end
+end
+
+struct SearchChannel
+  include DB::Serializable
+
+  property author : String
+  property ucid : String
+  property author_thumbnail : String
+  property subscriber_count : Int32
+  property video_count : Int32
+  property description_html : String
+  property auto_generated : Bool
+
+  def to_json(locale, json : JSON::Builder)
+    json.object do
+      json.field "type", "channel"
+      json.field "author", self.author
+      json.field "authorId", self.ucid
+      json.field "authorUrl", "/channel/#{self.ucid}"
+
+      json.field "authorThumbnails" do
+        json.array do
+          qualities = {32, 48, 76, 100, 176, 512}
+
+          qualities.each do |quality|
+            json.object do
+              json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+              json.field "width", quality
+              json.field "height", quality
+            end
+          end
+        end
+      end
+
+      json.field "autoGenerated", self.auto_generated
+      json.field "subCount", self.subscriber_count
+      json.field "videoCount", self.video_count
+
+      json.field "description", html_to_content(self.description_html)
+      json.field "descriptionHtml", self.description_html
+    end
+  end
+
+  def to_json(locale, json : JSON::Builder | Nil = nil)
+    if json
+      to_json(locale, json)
+    else
+      JSON.build do |json|
+        to_json(locale, json)
+      end
+    end
+  end
+end
+
+class Category
+  include DB::Serializable
+
+  property title : String
+  property contents : Array(SearchItem) | SearchItem
+  property browse_endpoint_data : String?
+  property continuation_token : String?
+  property badges : Array(Tuple(String, String))?
+
+  def to_json(locale, json : JSON::Builder)
+    json.object do
+      json.field "title", self.title
+      json.field "contents", self.contents
+    end
+  end
+
+  def to_json(locale, json : JSON::Builder | Nil = nil)
+    if json
+      to_json(locale, json)
+    else
+      JSON.build do |json|
+        to_json(locale, json)
+      end
+    end
+  end
+end
+
+alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

src/invidious/routes/channels.cr

@@ -102,7 +102,6 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute
       return env.redirect "/channel/#{channel.ucid}"
     end
 
-    # When a channel only has a single category it lacks the category param option so we'll handle it here.
     if continuation
       offset = env.params.query["offset"]?
       if offset

src/invidious/search.cr

@@ -1,235 +1,3 @@
-struct SearchVideo
-  include DB::Serializable
-
-  property title : String
-  property id : String
-  property author : String
-  property ucid : String
-  property published : Time
-  property views : Int64
-  property description_html : String
-  property length_seconds : Int32
-  property live_now : Bool
-  property paid : Bool
-  property premium : Bool
-  property premiere_timestamp : Time?
-
-  def to_xml(auto_generated, query_params, xml : XML::Builder)
-    query_params["v"] = self.id
-
-    xml.element("entry") do
-      xml.element("id") { xml.text "yt:video:#{self.id}" }
-      xml.element("yt:videoId") { xml.text self.id }
-      xml.element("yt:channelId") { xml.text self.ucid }
-      xml.element("title") { xml.text self.title }
-      xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
-
-      xml.element("author") do
-        if auto_generated
-          xml.element("name") { xml.text self.author }
-          xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
-        else
-          xml.element("name") { xml.text author }
-          xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
-        end
-      end
-
-      xml.element("content", type: "xhtml") do
-        xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
-          xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
-            xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
-          end
-
-          xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
-        end
-      end
-
-      xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
-
-      xml.element("media:group") do
-        xml.element("media:title") { xml.text self.title }
-        xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
-          width: "320", height: "180")
-        xml.element("media:description") { xml.text html_to_content(self.description_html) }
-      end
-
-      xml.element("media:community") do
-        xml.element("media:statistics", views: self.views)
-      end
-    end
-  end
-
-  def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
-    if xml
-      to_xml(HOST_URL, auto_generated, query_params, xml)
-    else
-      XML.build do |json|
-        to_xml(HOST_URL, auto_generated, query_params, xml)
-      end
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder)
-    json.object do
-      json.field "type", "video"
-      json.field "title", self.title
-      json.field "videoId", self.id
-
-      json.field "author", self.author
-      json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
-
-      json.field "videoThumbnails" do
-        generate_thumbnails(json, self.id)
-      end
-
-      json.field "description", html_to_content(self.description_html)
-      json.field "descriptionHtml", self.description_html
-
-      json.field "viewCount", self.views
-      json.field "published", self.published.to_unix
-      json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
-      json.field "lengthSeconds", self.length_seconds
-      json.field "liveNow", self.live_now
-      json.field "paid", self.paid
-      json.field "premium", self.premium
-      json.field "isUpcoming", self.is_upcoming
-
-      if self.premiere_timestamp
-        json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
-      end
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder | Nil = nil)
-    if json
-      to_json(locale, json)
-    else
-      JSON.build do |json|
-        to_json(locale, json)
-      end
-    end
-  end
-
-  def is_upcoming
-    premiere_timestamp ? true : false
-  end
-end
-
-struct SearchPlaylistVideo
-  include DB::Serializable
-
-  property title : String
-  property id : String
-  property length_seconds : Int32
-end
-
-struct SearchPlaylist
-  include DB::Serializable
-
-  property title : String
-  property id : String
-  property author : String
-  property ucid : String
-  property video_count : Int32
-  property videos : Array(SearchPlaylistVideo)
-  property thumbnail : String?
-
-  def to_json(locale, json : JSON::Builder)
-    json.object do
-      json.field "type", "playlist"
-      json.field "title", self.title
-      json.field "playlistId", self.id
-      json.field "playlistThumbnail", self.thumbnail
-
-      json.field "author", self.author
-      json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
-
-      json.field "videoCount", self.video_count
-      json.field "videos" do
-        json.array do
-          self.videos.each do |video|
-            json.object do
-              json.field "title", video.title
-              json.field "videoId", video.id
-              json.field "lengthSeconds", video.length_seconds
-
-              json.field "videoThumbnails" do
-                generate_thumbnails(json, video.id)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder | Nil = nil)
-    if json
-      to_json(locale, json)
-    else
-      JSON.build do |json|
-        to_json(locale, json)
-      end
-    end
-  end
-end
-
-struct SearchChannel
-  include DB::Serializable
-
-  property author : String
-  property ucid : String
-  property author_thumbnail : String
-  property subscriber_count : Int32
-  property video_count : Int32
-  property description_html : String
-  property auto_generated : Bool
-
-  def to_json(locale, json : JSON::Builder)
-    json.object do
-      json.field "type", "channel"
-      json.field "author", self.author
-      json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
-
-      json.field "authorThumbnails" do
-        json.array do
-          qualities = {32, 48, 76, 100, 176, 512}
-
-          qualities.each do |quality|
-            json.object do
-              json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
-              json.field "width", quality
-              json.field "height", quality
-            end
-          end
-        end
-      end
-
-      json.field "autoGenerated", self.auto_generated
-      json.field "subCount", self.subscriber_count
-      json.field "videoCount", self.video_count
-
-      json.field "description", html_to_content(self.description_html)
-      json.field "descriptionHtml", self.description_html
-    end
-  end
-
-  def to_json(locale, json : JSON::Builder | Nil = nil)
-    if json
-      to_json(locale, json)
-    else
-      JSON.build do |json|
-        to_json(locale, json)
-      end
-    end
-  end
-end
-
-alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
-
 def channel_search(query, page, channel)
   response = YT_POOL.client &.get("/channel/#{channel}")
 

src/invidious/views/channel/featured_channels.ecr

@@ -14,7 +14,7 @@
                 <details open="">
                     <summary style="display: revert;">
                         <h3 class="category-heading">
-                            <% if (category_request_param = category.browse_endpoint_param).is_a?(String) %>
+                            <% if (category_request_param = category.browse_endpoint_data).is_a?(String) %>
                                 <a href="/channel/<%=channel.ucid%>/channels/<%=HTML.escape(category_request_param)%>">
                                     <%= category.title %>
                                 </a>
@@ -25,8 +25,12 @@
                     </summary>
                     <% contents = category.contents%>
                     <div class="pure-g section-contents">
-                        <% if contents.is_a?(Array(FeaturedChannel)) %>
+                        <% if contents.is_a?(Array) %>
                             <% contents.each do |item|%>
+                                <% if !item.is_a?(SearchChannel)%>
+                                    <% next %>
+                                <% end %>
+
                                 <div class="channel-profile pure-u-1 pure-u-sm-1-2 pure-u-md-1-3 pure-u-lg-1-4 pure-u-xl-1-5">
                                     <a class="featured-channel-icon" href="/channel/<%= item.ucid %>">
                                         <% if !env.get("preferences").as(Preferences).thin_mode %>
@@ -47,7 +51,11 @@
                                     </div>
                                 </div>
                             <%end%>
-                        <% elsif contents.is_a?(FeaturedChannel) %>
+                        <% elsif contents.is_a?(SearchItem) %>
+                            <% if !contents.is_a?(SearchChannel)%>
+                                <% next %>
+                            <% end %>
+
                             <%item = contents %>
                             <div class="channel-profile large-featured-channel pure-u-1">
                                 <a class="featured-channel-icon" href="/channel/<%= item.ucid %>">
@@ -69,11 +77,10 @@
                                     <% sub_count_text = number_to_short_text(contents.subscriber_count) %>
                                     <%= rendered "components/subscribe_widget" %>
                             </div>
-                        <%end%>
+                        <% end %>
                     </div>
                 </details>
             </div>
-            </div>
         <% end %>
     <% else %>
         <h3 class="pure-u-1">

src/invidious/views/components/item.ecr

@@ -97,6 +97,7 @@
                     <%= item.responds_to?(:views) && item.views ? translate(locale, "`x` views", number_to_short_text(item.views || 0)) : "" %>
                 </div>
             </h5>
+        <% when Category %>
         <% else %>
             <% if !env.get("preferences").as(Preferences).thin_mode %>
                 <a style="width:100%" href="/watch?v=<%= item.id %>">