From b5921326f749cd2c776b472302cee3ca17d292e8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 2 Oct 2021 15:13:57 -0700 Subject: [PATCH] Rebase fixes --- src/invidious.cr | 1 - src/invidious/channel/about.cr | 210 ------------ src/invidious/channel/channels.cr | 319 ------------------ src/invidious/channel/community.cr | 275 --------------- src/invidious/channel/playlists.cr | 38 --- src/invidious/channel/videos.cr | 89 ----- .../featured_channels.cr | 6 +- src/invidious/{channel => channels}/home.cr | 2 +- src/invidious/routes/channels.cr | 24 +- src/invidious/views/watch.ecr | 4 +- 10 files changed, 17 insertions(+), 951 deletions(-) delete mode 100644 src/invidious/channel/about.cr delete mode 100644 src/invidious/channel/channels.cr delete mode 100644 src/invidious/channel/community.cr delete mode 100644 src/invidious/channel/playlists.cr delete mode 100644 src/invidious/channel/videos.cr rename src/invidious/{channel => channels}/featured_channels.cr (96%) rename src/invidious/{channel => channels}/home.cr (90%) diff --git a/src/invidious.cr b/src/invidious.cr index fab08352..ef763e94 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,7 +27,6 @@ require "yaml" require "compress/zip" require "protodec/utils" require "./invidious/helpers/**" -require "./invidious/channel/**" require "./invidious/*" require "./invidious/channels/*" require "./invidious/routes/**" diff --git a/src/invidious/channel/about.cr b/src/invidious/channel/about.cr deleted file mode 100644 index c82b4595..00000000 --- a/src/invidious/channel/about.cr +++ /dev/null @@ -1,210 +0,0 @@ -# TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable - - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property paid : Bool - property country : String - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) - property links : Array(Tuple(String, String, String)) -end - -def get_about_info(ucid, locale) - result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") - if result.status_code != 200 - result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") - end - - if md = result.headers["location"]?.try &.match(/\/channel\/(?UC[a-zA-Z0-9_-]{22})/) - raise ChannelRedirect.new(channel_id: md["ucid"]) - end - - if result.status_code != 200 - raise InfoException.new("This channel does not exist.") - end - - about = XML.parse_html(result.body) - if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) - raise InfoException.new("This channel does not exist.") - end - - initdata = extract_initial_data(result.body) - if initdata.empty? - error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip - error_message ||= translate(locale, "Could not get channel info.") - raise InfoException.new(error_message) - end - - if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? - raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) - end - - auto_generated = false - # Check for special auto generated gaming channels - if !initdata.has_key?("metadata") - auto_generated = true - end - - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - description = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]["simpleText"].as_s - description_html = HTML.escape(description).gsub("\n", "
") - - paid = false - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } - - related_channels = [] of AboutRelatedChannel - else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end - - description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || "" - description_html = HTML.escape(description).gsub("\n", "
") - - paid = about.xpath_node(%q(//meta[@itemprop="paid"])).not_nil!["content"] == "True" - is_family_friendly = about.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True" - allowed_regions = about.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",") - - related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] - .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? - .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| - renderer = node["miniChannelRenderer"]? - related_id = renderer.try &.["channelId"]?.try &.as_s? - related_id ||= "" - - related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? - related_title ||= "" - - related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? - .try &.["url"]?.try &.as_s? - related_author_url ||= "" - - related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? - related_author_thumbnails ||= [] of JSON::Any - - related_author_thumbnail = "" - if related_author_thumbnails.size > 0 - related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? - related_author_thumbnail ||= "" - end - - AboutRelatedChannel.new({ - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - }) - end - related_channels ||= [] of AboutRelatedChannel - end - - country = "" - total_views = 0_i64 - joined = Time.unix(0) - links = [] of {String, String, String} - - tabs = [] of String - - tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? - if !tabs_json.nil? - # Retrieve information from the tabs array. The index we are looking for varies between channels. - tabs_json.each do |node| - # Try to find the about section which is located in only one of the tabs. - channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? - .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? - .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? - - if !channel_about_meta.nil? - total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s } - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # External link parsing - channel_about_meta["primaryLinks"]?.try &.as_a.each do |link| - link_title = link["title"]["simpleText"].as_s - link_url = URI.parse(link["navigationEndpoint"]["urlEndpoint"]["url"].to_s) - link_icon_url = link["icon"]?.try &.["thumbnails"][0]["url"].to_s || "" - - if {"m.youtube.com", "www.youtube.com", "youtu.be"}.includes? link_url.host - if link_url.path == "/redirect" - link_url = HTTP::Params.parse(link_url.query.not_nil!)["q"] - else - link_url = link_url.request_target.to_s - end - else - link_url = link_url.to_s - end - - links << {link_title, link_url, link_icon_url} - end - - country = channel_about_meta["country"]?.try &.["simpleText"].as_s || "" - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && - (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" - auto_generated = true - end - end - end - tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map { |node| node["tabRenderer"]["title"].as_s.downcase } - end - - sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? - .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - - AboutChannel.new({ - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - paid: paid, - country: country, - total_views: total_views, - sub_count: sub_count, - joined: joined, - is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs, - links: links, - }) -end diff --git a/src/invidious/channel/channels.cr b/src/invidious/channel/channels.cr deleted file mode 100644 index 5b69ade1..00000000 --- a/src/invidious/channel/channels.cr +++ /dev/null @@ -1,319 +0,0 @@ -struct InvidiousChannel - include DB::Serializable - - property id : String - property author : String - property updated : Time - property deleted : Bool - property subscribed : Time? -end - -struct ChannelVideo - include DB::Serializable - - property id : String - property title : String - property published : Time - property updated : Time - property ucid : String - property author : String - property length_seconds : Int32 = 0 - property live_now : Bool = false - property premiere_timestamp : Time? = nil - property views : Int64? = nil - - def to_json(locale, json : JSON::Builder) - json.object do - json.field "type", "shortVideo" - - json.field "title", self.title - json.field "videoId", self.id - json.field "videoThumbnails" do - generate_thumbnails(json, self.id) - end - - json.field "lengthSeconds", self.length_seconds - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - json.field "published", self.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) - - json.field "viewCount", self.views - end - end - - def to_json(locale, json : JSON::Builder | Nil = nil) - if json - to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end - end - end - - def to_xml(locale, query_params, xml : XML::Builder) - query_params["v"] = self.id - - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") - - xml.element("author") do - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - end - end - end - - def to_xml(locale, xml : XML::Builder | Nil = nil) - if xml - to_xml(locale, xml) - else - XML.build do |xml| - to_xml(locale, xml) - end - end - end - - def to_tuple - {% begin %} - { - {{*@type.instance_vars.map { |var| var.name }}} - } - {% end %} - end -end - -struct AboutRelatedChannel - include DB::Serializable - - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - -class ChannelRedirect < Exception - property channel_id : String - - def initialize(@channel_id) - end -end - -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) - finished_channel = Channel(String | Nil).new - - spawn do - active_threads = 0 - active_channel = Channel(Nil).new - - channels.each do |ucid| - if active_threads >= max_threads - active_channel.receive - active_threads -= 1 - end - - active_threads += 1 - spawn do - begin - get_channel(ucid, db, refresh, pull_all_videos) - finished_channel.send(ucid) - rescue ex - finished_channel.send(nil) - ensure - active_channel.send(nil) - end - end - end - end - - final = [] of String - channels.size.times do - if ucid = finished_channel.receive - final << ucid - end - end - - return final -end - -def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) - if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) - end - else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) - end - - return channel -end - -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) - LOGGER.debug("fetch_channel: #{ucid}") - LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") - - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse_html(rss) - - author = rss.xpath_node(%q(//feed/title)) - if !author - raise InfoException.new("Deleted or invalid channel") - end - author = author.content - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end - - LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - - page = 1 - - LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) - - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//feed/entry").each do |entry| - video_id = entry.xpath_node("videoid").not_nil!.content - title = entry.xpath_node("title").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - author = entry.xpath_node("author/name").not_nil!.content - ucid = entry.xpath_node("channelid").not_nil!.content - views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? - views ||= 0_i64 - - channel_video = videos.select { |video| video.id == video_id }[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.live_now - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - - video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: Time.utc, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, - }) - - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") - - # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) - else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") - end - end - - if pull_all_videos - page += 1 - - ids = [] of String - - loop do - initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - videos = extract_videos(initial_data, author, ucid) - - count = videos.size - videos = videos.map { |video| ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) } - - videos.each do |video| - ids << video.id - - # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, - # so since they don't provide a published date here we can safely ignore them. - if Time.utc - video.published > 1.minute - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert - end - end - - break if count < 25 - page += 1 - end - end - - channel = InvidiousChannel.new({ - id: ucid, - author: author, - updated: Time.utc, - deleted: false, - subscribed: nil, - }) - - return channel -end diff --git a/src/invidious/channel/community.cr b/src/invidious/channel/community.cr deleted file mode 100644 index 97ab30ec..00000000 --- a/src/invidious/channel/community.cr +++ /dev/null @@ -1,275 +0,0 @@ -# TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, format, thin_mode) - response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") - if response.status_code != 200 - response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") - end - - if response.status_code != 200 - raise InfoException.new("This channel does not exist.") - end - - ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] - - if !continuation || continuation.empty? - initial_data = extract_initial_data(response.body) - body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? - - if !body - raise InfoException.new("Could not extract community tab.") - end - - body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] - else - continuation = produce_channel_community_continuation(ucid, continuation) - - headers = HTTP::Headers.new - headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - - session_token = response.body.match(/"XSRF_TOKEN":"(?[^"]+)"/).try &.["session_token"]? || "" - post_req = { - session_token: session_token, - } - - response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) - body = JSON.parse(response.body) - - body = body["response"]["continuationContents"]["itemSectionContinuation"]? || - body["response"]["continuationContents"]["backstageCommentsContinuation"]? - - if !body - raise InfoException.new("Could not extract continuation.") - end - end - - continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s - posts = body["contents"].as_a - - if message = posts[0]["messageRenderer"]? - error_message = (message["text"]["simpleText"]? || - message["text"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s || "" - raise InfoException.new(error_message) - end - - response = JSON.build do |json| - json.object do - json.field "authorId", ucid - json.field "comments" do - json.array do - posts.each do |post| - comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || - post["backstageCommentsContinuation"]? - - post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || - post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? - - next if !post - - content_html = post["contentText"]?.try { |t| parse_content(t) } || "" - author = post["authorText"]?.try &.["simpleText"]? || "" - - json.object do - json.field "author", author - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s - - qualities.each do |quality| - json.object do - json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - - if post["authorEndpoint"]? - json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - else - json.field "authorId", "" - json.field "authorUrl", "" - end - - published_text = post["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) - - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end - - like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] - .try &.as_s.gsub(/\D/, "").to_i? || 0 - - json.field "content", html_to_content(content_html) - json.field "contentHtml", content_html - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - json.field "likeCount", like_count - json.field "commentId", post["postId"]? || post["commentId"]? || "" - json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid - - if attachment = post["backstageAttachment"]? - json.field "attachment" do - json.object do - case attachment.as_h - when .has_key?("videoRenderer") - attachment = attachment["videoRenderer"] - json.field "type", "video" - - if !attachment["videoId"]? - error_message = (attachment["title"]["simpleText"]? || - attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?) - - json.field "error", error_message - else - video_id = attachment["videoId"].as_s - - video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? - json.field "title", video_title - json.field "videoId", video_id - json.field "videoThumbnails" do - generate_thumbnails(json, video_id) - end - - json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) - - author_info = attachment["ownerText"]["runs"][0].as_h - - json.field "author", author_info["text"].as_s - json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - - # TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers" - # TODO: json.field "authorVerified", "ownerBadges" - - published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s) - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 - - json.field "viewCount", view_count - json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) - end - when .has_key?("backstageImageRenderer") - attachment = attachment["backstageImageRenderer"] - json.field "type", "image" - - json.field "imageThumbnails" do - json.array do - thumbnail = attachment["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - - qualities = {320, 560, 640, 1280, 2000} - - qualities.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i - end - end - end - end - # TODO - # when .has_key?("pollRenderer") - # attachment = attachment["pollRenderer"] - # json.field "type", "poll" - else - json.field "type", "unknown" - json.field "error", "Unrecognized attachment type." - end - end - end - end - - if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || - comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i?) - continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s - continuation ||= "" - - json.field "replies" do - json.object do - json.field "replyCount", reply_count - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - end - end - end - end - - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - - if format == "html" - response = JSON.parse(response) - content_html = template_youtube_comments(response, locale, thin_mode) - - response = JSON.build do |json| - json.object do - json.field "contentHtml", content_html - end - end - end - - return response -end - -def produce_channel_community_continuation(ucid, cursor) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => cursor || "", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def extract_channel_community_cursor(continuation) - object = URI.decode_www_form(continuation) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try { |i| i["80226972:0:embedded"]["3:1:base64"].as_h } - - if object["53:2:embedded"]?.try &.["3:0:embedded"]? - object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] - .try { |i| i["2:0:base64"].as_h } - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i, padding: false) } - - object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") - end - - cursor = Protodec::Any.cast_json(object) - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - - cursor -end diff --git a/src/invidious/channel/playlists.cr b/src/invidious/channel/playlists.cr deleted file mode 100644 index 9180cbeb..00000000 --- a/src/invidious/channel/playlists.cr +++ /dev/null @@ -1,38 +0,0 @@ -def fetch_channel_playlists(ucid, author, continuation, sort_by) - if continuation - response_json = request_youtube_api_browse(continuation) - continuationItems = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem, nil if !continuationItems - - items = [] of SearchItem - continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| - extract_item(item, author, ucid).try { |t| items << t } - } - - continuation = continuationItems.as_a.last["continuationItemRenderer"]? - .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s - else - url = "/channel/#{ucid}/playlists?flow=list&view=1" - - case sort_by - when "last", "last_added" - # - when "oldest", "oldest_created" - url += "&sort=da" - when "newest", "newest_created" - url += "&sort=dd" - else nil # Ignore - end - - response = YT_POOL.client &.get(url) - initial_data = extract_initial_data(response.body) - return [] of SearchItem, nil if !initial_data - - items = extract_items(initial_data, author, ucid) - continuation = response.body.match(/"token":"(?[^"]+)"/).try &.["continuation"]? - end - - return items, continuation -end diff --git a/src/invidious/channel/videos.cr b/src/invidious/channel/videos.cr deleted file mode 100644 index cc291e9e..00000000 --- a/src/invidious/channel/videos.cr +++ /dev/null @@ -1,89 +0,0 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "videos", - "6:varint" => 2_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, - }, - }, - } - - if !v2 - if auto_generated - seed = Time.unix(1525757349) - until seed >= Time.utc - seed += 1.month - end - timestamp = seed - (page - 1).months - - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" - end - else - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - - object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ - "1:varint" => 30_i64 * (page - 1), - }))), - }))) - end - - case sort_by - when "newest" - when "popular" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64 - when "oldest" - object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64 - else nil # Ignore - end - - object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) - object["80226972:embedded"].delete("3:base64") - - continuation = object.try { |i| Protodec::Any.cast_json(object) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - continuation = produce_channel_videos_continuation(ucid, page, - auto_generated: auto_generated, sort_by: sort_by, v2: true) - - return request_youtube_api_browse(continuation) -end - -def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - videos = [] of SearchVideo - - 2.times do |i| - initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - videos.concat extract_videos(initial_data, author, ucid) - end - - return videos.size, videos -end - -def get_latest_videos(ucid) - initial_data = get_channel_videos_response(ucid) - author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s - - return extract_videos(initial_data, author, ucid) -end - -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end diff --git a/src/invidious/channel/featured_channels.cr b/src/invidious/channels/featured_channels.cr similarity index 96% rename from src/invidious/channel/featured_channels.cr rename to src/invidious/channels/featured_channels.cr index 860f25e6..03c18309 100644 --- a/src/invidious/channel/featured_channels.cr +++ b/src/invidious/channels/featured_channels.cr @@ -2,7 +2,7 @@ # # Returned as an array of Category objects containing different channels. def fetch_channel_featured_channels(ucid) : Array(Category) - initial_data = request_youtube_api_browse(ucid, "EghjaGFubmVscw%3D%3D") + initial_data = YoutubeAPI.browse(ucid, params: "EghjaGFubmVscw%3D%3D") channels_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) @@ -72,7 +72,7 @@ def fetch_selected_channel_featuring_category(ucid, view, shelf_id) : Tuple(Cate category_url = "/channel/#{ucid}/channels?view=#{view}&shelf_id=#{shelf_id}" params = produce_featured_channel_browse_param(view.to_i64, shelf_id.to_i64) - initial_data = request_youtube_api_browse(ucid, params) + initial_data = YoutubeAPI.browse(ucid, params: params) channels_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) continuation_token = fetch_continuation_token(initial_data) @@ -101,7 +101,7 @@ end # TODO: The query_title here is really only used for frontend rendering. # And since it's a URL parameter we should be able to just request it directly within the template files. def fetch_channel_featured_channels_category_continuation(continuation, query_title) : Tuple(Category, String | Nil) - initial_data = request_youtube_api_browse(continuation) + initial_data = YoutubeAPI.browse(continuation) items = extract_items(initial_data) continuation_token = fetch_continuation_token(initial_data) diff --git a/src/invidious/channel/home.cr b/src/invidious/channels/home.cr similarity index 90% rename from src/invidious/channel/home.cr rename to src/invidious/channels/home.cr index ff872df5..ec6544e3 100644 --- a/src/invidious/channel/home.cr +++ b/src/invidious/channels/home.cr @@ -1,5 +1,5 @@ def fetch_channel_home(ucid, channel) - initial_data = request_youtube_api_browse(ucid, "EghmZWF0dXJlZA%3D%3D") + initial_data = YoutubeAPI.browse(ucid, params: "EghmZWF0dXJlZA%3D%3D") items = extract_items(initial_data, channel.author, channel.ucid) # Channel trailer needs some slight special handling diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index bdaabbcd..26b23d42 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,14 @@ {% skip_file if flag?(:api_only) %} module Invidious::Routes::Channels + def self.channel(env) + if !env.get("preferences").as(Preferences).view_channel_homepage_by_default + env.redirect "/channel/#{env.params.url["ucid"]}/videos" + else + env.redirect "/channel/#{env.params.url["ucid"]}/home" + end + end + def self.home(env) data = self.fetch_basic_information(env) if !data.is_a?(Tuple) @@ -60,6 +68,7 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data + sort_options = {"last", "oldest", "newest"} sort_by = env.params.query["sort_by"]?.try &.downcase sort_by ||= "last" @@ -180,17 +189,6 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data - ucid = env.params.url["ucid"] - continuation = env.params.query["continuation"]? - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_template(500, ex) - end - templated "channel/about", buffer_footer: true end @@ -240,11 +238,11 @@ module Invidious::Routes::Channels end end - private def search(env) + def self.search(env) return env.redirect "/search?#{env.params.query}&channel=#{env.params.url["ucid"]}" end - private def fetch_basic_information(env) + private def self.fetch_basic_information(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? user = env.get? "user" diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 418f85c5..83bc8af6 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -190,8 +190,8 @@ we're going to need to do it here in order to allow for translations. <% end %> <% captions.each do |caption| %> - <% end %>