From c9594d46afb1f4a1d8fd75be5ef98309a92c2761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Sun, 13 Mar 2022 22:46:45 +0100 Subject: [PATCH 1/4] Add links redirect inside channel description --- src/invidious/helpers/utils.cr | 8 ++++++++ src/invidious/routes/channels.cr | 6 ++++-- src/invidious/views/channel.ecr | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8ae5034a..4f2df699 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -391,3 +391,11 @@ def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = " end return str end + +def make_html_with_links(baseText : String) : String + returnValue = baseText.dup + returnValue.scan(/https?:\/\/[^ \n]*/).each do |match| + returnValue = returnValue.sub(match[0], "#{match[0]}") + end + return returnValue +end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index cd2e3323..88c394c2 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -10,7 +10,7 @@ module Invidious::Routes::Channels if !data.is_a?(Tuple) return data end - locale, user, subscriptions, continuation, ucid, channel = data + locale, user, subscriptions, continuation, ucid, channel, description = data page = env.params.query["page"]?.try &.to_i? page ||= 1 @@ -201,6 +201,8 @@ module Invidious::Routes::Channels return error_template(500, ex) end - return {locale, user, subscriptions, continuation, ucid, channel} + description = make_html_with_links(channel.description_html) + + return {locale, user, subscriptions, continuation, ucid, channel, description} end end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 92f81ee4..b8a14afa 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -32,7 +32,7 @@
-

<%= channel.description_html %>

+

<%= description %>

From 28efeaa4f2cd3c3d13e0d865dc50a2ec0be8e054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Mon, 14 Mar 2022 22:37:22 +0100 Subject: [PATCH 2/4] Update management of channel description Follow this comment : https://github.com/iv-org/invidious/pull/2968#issuecomment-1066428317 --- src/invidious/channels/about.cr | 20 ++++++++++++++++---- src/invidious/comments.cr | 26 ++++++++++++++++++++++++++ src/invidious/helpers/utils.cr | 8 -------- src/invidious/routes/channels.cr | 6 ++---- src/invidious/views/channel.ecr | 2 +- 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index d48fd1fb..da71e9a8 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -6,6 +6,7 @@ record AboutChannel, author_url : String, author_thumbnail : String, banner : String?, + description : String, description_html : String, total_views : Int64, sub_count : Int32, @@ -52,8 +53,7 @@ def get_about_info(ucid, locale) : AboutChannel banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? banner = banners.try &.[-1]?.try &.["url"].as_s? - description = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]["simpleText"].as_s - description_html = HTML.escape(description) + description_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) @@ -75,13 +75,24 @@ def get_about_info(ucid, locale) : AboutChannel author_verified_badge = initdata["header"].dig?("c4TabbedHeaderRenderer", "badges", 0, "metadataBadgeRenderer", "tooltip") author_verified = (author_verified_badge && author_verified_badge == "Verified") - description = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?.try &.as_s? || "" - description_html = HTML.escape(description) + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) end + description = !description_node.nil? ? description_node.as_s : "" + description_html = HTML.escape(description) + if !description_node.nil? + if description_node.as_h?.nil? + description_node = text_to_parsed_content(description_node.as_s) + end + description_html = parse_content(description_node) + if description_html == "" && description != "" + description_html = HTML.escape(description) + end + end + total_views = 0_i64 joined = Time.unix(0) @@ -125,6 +136,7 @@ def get_about_info(ucid, locale) : AboutChannel author_url: author_url, author_thumbnail: author_thumbnail, banner: banner, + description: description, description_html: description_html, total_views: total_views, sub_count: sub_count, diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 1f8de657..c1bcc0a6 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -560,6 +560,32 @@ def fill_links(html, scheme, host) return html.to_xml(options: XML::SaveOptions::NO_DECL) end +def text_to_parsed_content(text : String) : JSON::Any + nodes = [] of JSON::Any + text.split('\n').each do |line| + currentNodes = [] of JSON::Any + initialNode = {"text" => line} + currentNodes << (JSON.parse(initialNode.to_json)) + line.scan(/https?:\/\/[^ ]*/).each do |uriMatch| + lastNode = currentNodes[currentNodes.size - 1].as_h + splittedLastNode = lastNode["text"].as_s.split(uriMatch[0]) + lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) + currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + currentNode = {"text" => uriMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => uriMatch[0]}}} + currentNodes << (JSON.parse(currentNode.to_json)) + afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""} + currentNodes << (JSON.parse(afterNode.to_json)) + end + lastNode = currentNodes[currentNodes.size - 1].as_h + lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json) + currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + currentNodes.each do |node| + nodes << (node) + end + end + return JSON.parse({"runs" => nodes}.to_json) +end + def parse_content(content : JSON::Any, video_id : String? = "") : String content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
") } || "" diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 4f2df699..8ae5034a 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -391,11 +391,3 @@ def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = " end return str end - -def make_html_with_links(baseText : String) : String - returnValue = baseText.dup - returnValue.scan(/https?:\/\/[^ \n]*/).each do |match| - returnValue = returnValue.sub(match[0], "#{match[0]}") - end - return returnValue -end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 88c394c2..cd2e3323 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -10,7 +10,7 @@ module Invidious::Routes::Channels if !data.is_a?(Tuple) return data end - locale, user, subscriptions, continuation, ucid, channel, description = data + locale, user, subscriptions, continuation, ucid, channel = data page = env.params.query["page"]?.try &.to_i? page ||= 1 @@ -201,8 +201,6 @@ module Invidious::Routes::Channels return error_template(500, ex) end - description = make_html_with_links(channel.description_html) - - return {locale, user, subscriptions, continuation, ucid, channel, description} + return {locale, user, subscriptions, continuation, ucid, channel} end end diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index b8a14afa..92f81ee4 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -32,7 +32,7 @@
-

<%= description %>

+

<%= channel.description_html %>

From 2e195575a62886d2492e9c505d7bc283b7f60097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Mon, 18 Apr 2022 17:20:47 +0200 Subject: [PATCH 3/4] Rename uriMatch to urlMatch inside comments.cr This refactor update text_to_parsed_content method --- src/invidious/comments.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index c1bcc0a6..199e4be3 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -566,12 +566,12 @@ def text_to_parsed_content(text : String) : JSON::Any currentNodes = [] of JSON::Any initialNode = {"text" => line} currentNodes << (JSON.parse(initialNode.to_json)) - line.scan(/https?:\/\/[^ ]*/).each do |uriMatch| + line.scan(/https?:\/\/[^ ]*/).each do |urlMatch| lastNode = currentNodes[currentNodes.size - 1].as_h - splittedLastNode = lastNode["text"].as_s.split(uriMatch[0]) + splittedLastNode = lastNode["text"].as_s.split(urlMatch[0]) lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) - currentNode = {"text" => uriMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => uriMatch[0]}}} + currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} currentNodes << (JSON.parse(currentNode.to_json)) afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""} currentNodes << (JSON.parse(afterNode.to_json)) From d8fb4f0a87f9fad3c32dd8528b789008945369a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9ry=20Mathieu=20=28Mathius=29?= Date: Mon, 18 Apr 2022 17:29:04 +0200 Subject: [PATCH 4/4] Update text_to_parsed_content for add docs Follow this comment : https://github.com/iv-org/invidious/pull/2968#discussion_r851808433 --- src/invidious/comments.cr | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 199e4be3..83074098 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -562,23 +562,39 @@ end def text_to_parsed_content(text : String) : JSON::Any nodes = [] of JSON::Any + # For each line convert line to array of nodes text.split('\n').each do |line| + # In first case line is just a simple node before + # check patterns inside line + # { 'text': line } currentNodes = [] of JSON::Any initialNode = {"text" => line} currentNodes << (JSON.parse(initialNode.to_json)) + + # For each match with url pattern, get last node and preserve + # last node before create new node with url information + # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } line.scan(/https?:\/\/[^ ]*/).each do |urlMatch| + # Retrieve last node and update node without match lastNode = currentNodes[currentNodes.size - 1].as_h splittedLastNode = lastNode["text"].as_s.split(urlMatch[0]) lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + # Create new node with match and navigation infos currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} currentNodes << (JSON.parse(currentNode.to_json)) + # If text remain after match create new simple node with text after match afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""} currentNodes << (JSON.parse(afterNode.to_json)) end + + # After processing of matches inside line + # Add \n at end of last node for preserve carriage return lastNode = currentNodes[currentNodes.size - 1].as_h lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json) currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + + # Finally add final nodes to nodes returned currentNodes.each do |node| nodes << (node) end