From af592ea8c15eeb985edecd875c66b05486bf7f89 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 11 Jul 2019 07:27:42 -0500 Subject: [PATCH] Fix extraction for ytInitialData --- src/invidious/channels.cr | 4 ++-- src/invidious/helpers/helpers.cr | 9 +++++++++ src/invidious/mixes.cr | 13 ++++--------- src/invidious/trending.cr | 9 ++------- src/invidious/videos.cr | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 858fbded..0bbd0eaa 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -636,8 +636,8 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] if !continuation || continuation.empty? - response = JSON.parse(response.body.match(/window\["ytInitialData"\] = (?.*?);\n/).try &.["info"] || "{}") - body = response["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? + initial_data = extract_initial_data(response.body) + body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? if !body raise "Could not extract community tab." diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 43255972..7f7725c1 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -826,3 +826,12 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi connection_channel.send({false, connection}) end end + +def extract_initial_data(body) + initial_data = body.match(/window\["ytInitialData"\] = (?.*?);\n/).try &.["info"] || "{}" + if initial_data.starts_with?("JSON.parse(\"") + return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s) + else + return JSON.parse(initial_data) + end +end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 8664f83d..a1d592f0 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -28,18 +28,13 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) end response = client.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers) - yt_data = response.body.match(/window\["ytInitialData"\] = (?.*);/) - if yt_data - yt_data = JSON.parse(yt_data["data"].rchop(";")) - else + initial_data = extract_initial_data(response.body) + + if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? raise translate(locale, "Could not create mix.") end - if !yt_data["contents"]["twoColumnWatchNextResults"]["playlist"]? - raise translate(locale, "Could not create mix.") - end - - playlist = yt_data["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"] + playlist = initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"] mix_title = playlist["title"].as_s contents = playlist["contents"].as_a diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 5f9d7920..32908157 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -14,14 +14,9 @@ def fetch_trending(trending_type, region, locale) response = client.get("/feed/trending?gl=#{region}&hl=en", headers).body - yt_data = response.match(/window\["ytInitialData"\] = (?.*);/) - if yt_data - yt_data = JSON.parse(yt_data["data"].rchop(";")) - else - raise translate(locale, "Could not pull trending pages.") - end + initial_data = extract_initial_data(response) - tabs = yt_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["subMenu"]["channelListSubMenuRenderer"]["contents"].as_a + tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["subMenu"]["channelListSubMenuRenderer"]["contents"].as_a url = tabs.select { |tab| tab["channelListSubMenuAvatarRenderer"]["title"]["simpleText"] == trending_type }[0]? if url diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index b97c8f94..41bf70df 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -925,7 +925,7 @@ def extract_polymer_config(body, html) end end - initial_data = JSON.parse(body.match(/window\["ytInitialData"\] = (?.*?);\n/).try &.["info"] || "{}") + initial_data = extract_initial_data(body) primary_results = initial_data["contents"]? .try &.["twoColumnWatchNextResults"]?