From ec0ddcd19dbdb2aaff696866fe758089d33355a7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 25 Jul 2021 04:24:31 -0700 Subject: [PATCH] Update community.cr and views to use new parser --- src/invidious/channels/community.cr | 242 +----------------- .../youtube/renderers/community_post.cr | 4 +- src/invidious/helpers/extractors.cr | 19 +- src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/routes/channels.cr | 20 +- src/invidious/views/community.ecr | 52 ++-- 6 files changed, 57 insertions(+), 282 deletions(-) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 97ab30ec..304eefc8 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -1,237 +1,13 @@ -# TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, format, thin_mode) - response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") - if response.status_code != 200 - response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") - end +# Fetches channel community posts for the initial page +def fetch_channel_community(ucid) + initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D") + return extract_items(initial_data) +end - if response.status_code != 200 - raise InfoException.new("This channel does not exist.") - end - - ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] - - if !continuation || continuation.empty? - initial_data = extract_initial_data(response.body) - body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? - - if !body - raise InfoException.new("Could not extract community tab.") - end - - body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] - else - continuation = produce_channel_community_continuation(ucid, continuation) - - headers = HTTP::Headers.new - headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - - session_token = response.body.match(/"XSRF_TOKEN":"(?[^"]+)"/).try &.["session_token"]? || "" - post_req = { - session_token: session_token, - } - - response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) - body = JSON.parse(response.body) - - body = body["response"]["continuationContents"]["itemSectionContinuation"]? || - body["response"]["continuationContents"]["backstageCommentsContinuation"]? - - if !body - raise InfoException.new("Could not extract continuation.") - end - end - - continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s - posts = body["contents"].as_a - - if message = posts[0]["messageRenderer"]? - error_message = (message["text"]["simpleText"]? || - message["text"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s || "" - raise InfoException.new(error_message) - end - - response = JSON.build do |json| - json.object do - json.field "authorId", ucid - json.field "comments" do - json.array do - posts.each do |post| - comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || - post["backstageCommentsContinuation"]? - - post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || - post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? - - next if !post - - content_html = post["contentText"]?.try { |t| parse_content(t) } || "" - author = post["authorText"]?.try &.["simpleText"]? || "" - - json.object do - json.field "author", author - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s - - qualities.each do |quality| - json.object do - json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality - end - end - end - end - - if post["authorEndpoint"]? - json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - else - json.field "authorId", "" - json.field "authorUrl", "" - end - - published_text = post["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) - - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end - - like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] - .try &.as_s.gsub(/\D/, "").to_i? || 0 - - json.field "content", html_to_content(content_html) - json.field "contentHtml", content_html - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - json.field "likeCount", like_count - json.field "commentId", post["postId"]? || post["commentId"]? || "" - json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid - - if attachment = post["backstageAttachment"]? - json.field "attachment" do - json.object do - case attachment.as_h - when .has_key?("videoRenderer") - attachment = attachment["videoRenderer"] - json.field "type", "video" - - if !attachment["videoId"]? - error_message = (attachment["title"]["simpleText"]? || - attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?) - - json.field "error", error_message - else - video_id = attachment["videoId"].as_s - - video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? - json.field "title", video_title - json.field "videoId", video_id - json.field "videoThumbnails" do - generate_thumbnails(json, video_id) - end - - json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) - - author_info = attachment["ownerText"]["runs"][0].as_h - - json.field "author", author_info["text"].as_s - json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - - # TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers" - # TODO: json.field "authorVerified", "ownerBadges" - - published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s) - - json.field "published", published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - - view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 - - json.field "viewCount", view_count - json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) - end - when .has_key?("backstageImageRenderer") - attachment = attachment["backstageImageRenderer"] - json.field "type", "image" - - json.field "imageThumbnails" do - json.array do - thumbnail = attachment["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - - qualities = {320, 560, 640, 1280, 2000} - - qualities.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i - end - end - end - end - # TODO - # when .has_key?("pollRenderer") - # attachment = attachment["pollRenderer"] - # json.field "type", "poll" - else - json.field "type", "unknown" - json.field "error", "Unrecognized attachment type." - end - end - end - end - - if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || - comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i?) - continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s - continuation ||= "" - - json.field "replies" do - json.object do - json.field "replyCount", reply_count - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - end - end - end - end - - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - json.field "continuation", extract_channel_community_cursor(continuation) - end - end - end - - if format == "html" - response = JSON.parse(response) - content_html = template_youtube_comments(response, locale, thin_mode) - - response = JSON.build do |json| - json.object do - json.field "contentHtml", content_html - end - end - end - - return response +# Fetches the next batch of community posts after the given cursor +def fetch_channel_community(ucid, cursor) + continuation = produce_channel_community_continuation(ucid, cursor) + initial_data = YoutubeAPI.browse(continuation) end def produce_channel_community_continuation(ucid, cursor) diff --git a/src/invidious/data_structs/youtube/renderers/community_post.cr b/src/invidious/data_structs/youtube/renderers/community_post.cr index 05ac191d..b883ee81 100644 --- a/src/invidious/data_structs/youtube/renderers/community_post.cr +++ b/src/invidious/data_structs/youtube/renderers/community_post.cr @@ -34,7 +34,7 @@ module YouTubeStructs # Community post data property post_id : String - property contents : String + property content_html : String property attachment : (VideoRenderer | PlaylistRenderer | CommunityPoll | String)? # string is image/gif property likes : Int32 property published : Time @@ -48,7 +48,7 @@ module YouTubeStructs json.field "author_thumbnail", self.author_thumbnail json.field "authorUrl", "/channel/#{self.author_id}" - json.field "contents", self.contents + json.field "contents", html_to_content(self.content_html) json.field "attachment", self.attachment.to_json json.field "likes", self.likes json.field "published", self.published.to_unix diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index c8cacc97..919f9f57 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -336,9 +336,7 @@ private module Parsers author_id = item_contents.dig("authorEndpoint", "browseEndpoint", "browseId").as_s author_thumbnail = item_contents.dig("authorThumbnail", "thumbnails", -1, "url").as_s # last item is highest quality - contents = String.build do |content_text| - item_contents["contentText"]["runs"].as_a.each { |t| content_text << t["text"] } - end + contents = item_contents["contentText"].try { |t| parse_content(t) } attachment_container = item_contents["backstageAttachment"]? @@ -365,14 +363,13 @@ private module Parsers author: author_name, author_id: author_id, author_thumbnail: author_thumbnail, - - post_id: post_id, - contents: contents, - attachment: attachment, - likes: likes, - published: published, - }) - end + post_id: post_id, + content_html: contents, + attachment: attachment, + likes: likes, + published: published, + }) + end end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 9fe9601c..480112a1 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -235,7 +235,7 @@ module Invidious::Routes::API::V1::Channels # sort_by = env.params.query["sort_by"]?.try &.downcase begin - fetch_channel_community(ucid, continuation, locale, format, thin_mode) + # fetch_channel_community(ucid, continuation, locale, format, thin_mode) rescue ex return error_json(500, ex) end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 2f94f24e..eb488e56 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -70,25 +70,19 @@ module Invidious::Routes::Channels end locale, user, subscriptions, continuation, ucid, channel = data - thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode - thin_mode = thin_mode == "true" - - continuation = env.params.query["continuation"]? - # sort_by = env.params.query["sort_by"]?.try &.downcase - if !channel.tabs.includes? "community" return env.redirect "/channel/#{channel.ucid}" end - begin - items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) - rescue ex : InfoException - env.response.status_code = 500 - error_message = ex.message - rescue ex - return error_template(500, ex) + if continuation + items = fetch_channel_community(ucid, continuation) + else + items = fetch_channel_community(ucid) end + pp items + + return env.redirect "/channel/#{channel.ucid}" if !items || items.empty? templated "community" end diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 15d8ed1e..d58db685 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -67,26 +67,34 @@
-<% if error_message %> -
-

<%= error_message %>

-
-<% else %> -
- <%= template_youtube_comments(items.not_nil!, locale, thin_mode) %> -
-<% end %> +<%- items.each do |thread| -%> + <% next if !thread.is_a? YouTubeStructs::CommunityPost %> +
+
+ +
+ + +
+

><%= thread.author %>

+

<%= thread.content_html -%>

+ <% # Handles attachments: %> + <% attachment = thread.attachment %> + <% if attachment.is_a? YouTubeStructs::VideoRenderer %> +
+
+ +
+
+ <% elsif attachment.is_a? YouTubeStructs::PlaylistRenderer %> + <% elsif attachment.is_a? YouTubeStructs::CommunityPoll %> + <% elsif attachment.nil? %> + <% else %> + + <% end %> + +

<%= thread.likes %>

+
+
+<%- end%> - -