Captions: Add ability to use Innertube's transcripts API (#4001)

This commit is contained in:
Samantaz Fox 2023-09-18 23:31:56 +02:00
commit 842e9fade5
No known key found for this signature in database
GPG Key ID: F42821059186176E
9 changed files with 289 additions and 132 deletions

View File

@ -161,6 +161,19 @@ https_only: false
#force_resolve: #force_resolve:
##
## Use Innertube's transcripts API instead of timedtext for closed captions
##
## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567
##
## Subtitle experience may differ slightly on Invidious.
##
## Accepted values: true, false
## Default: false
##
# use_innertube_for_captions: false
# ----------------------------- # -----------------------------
# Logging # Logging
# ----------------------------- # -----------------------------

View File

@ -127,6 +127,9 @@ class Config
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100 property pool_size : Int32 = 100
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format # Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)] @[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new property cookies : HTTP::Cookies = HTTP::Cookies.new

View File

@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any)) getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any)) getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any)) getter audio_streams : Array(Hash(String, JSON::Any))
getter captions : Array(Invidious::Videos::Caption) getter captions : Array(Invidious::Videos::Captions::Metadata)
def initialize( def initialize(
@full_videos, @full_videos,

View File

@ -87,6 +87,13 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0] caption = caption[0]
end end
if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
initial_data = YoutubeAPI.get_transcript(params)
webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
else
# Timedtext API handling
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
# Auto-generated captions often have cues that aren't aligned properly with the video, # Auto-generated captions often have cues that aren't aligned properly with the video,
@ -153,6 +160,7 @@ module Invidious::Routes::API::V1::Videos
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end end
end end
end
if title = env.params.query["title"]? if title = env.params.query["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/

View File

@ -24,7 +24,7 @@ struct Video
property updated : Time property updated : Time
@[DB::Field(ignore: true)] @[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Caption @captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)] @[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))? property adaptive_fmts : Array(Hash(String, JSON::Any))?
@ -215,9 +215,9 @@ struct Video
keywords.includes? "YouTube Red" keywords.includes? "YouTube Red"
end end
def captions : Array(Invidious::Videos::Caption) def captions : Array(Invidious::Videos::Captions::Metadata)
if @captions.empty? && @info.has_key?("captions") if @captions.empty? && @info.has_key?("captions")
@captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
end end
return @captions return @captions

View File

@ -1,21 +1,24 @@
require "json" require "json"
module Invidious::Videos module Invidious::Videos
struct Caption module Captions
struct Metadata
property name : String property name : String
property language_code : String property language_code : String
property base_url : String property base_url : String
def initialize(@name, @language_code, @base_url) property auto_generated : Bool
def initialize(@name, @language_code, @base_url, @auto_generated)
end end
# Parse the JSON structure from Youtube # Parse the JSON structure from Youtube
def self.from_yt_json(container : JSON::Any) : Array(Caption) def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
caption_tracks = container caption_tracks = container
.dig?("playerCaptionsTracklistRenderer", "captionTracks") .dig?("playerCaptionsTracklistRenderer", "captionTracks")
.try &.as_a .try &.as_a
captions_list = [] of Caption captions_list = [] of Captions::Metadata
return captions_list if caption_tracks.nil? return captions_list if caption_tracks.nil?
caption_tracks.each do |caption| caption_tracks.each do |caption|
@ -25,7 +28,9 @@ module Invidious::Videos
language_code = caption["languageCode"].to_s language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s base_url = caption["baseUrl"].to_s
captions_list << Caption.new(name, language_code, base_url) auto_generated = (caption["kind"]? == "asr")
captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
end end
return captions_list return captions_list
@ -96,6 +101,7 @@ module Invidious::Videos
end end
return result return result
end end
end
# List of all caption languages available on Youtube. # List of all caption languages available on Youtube.
LANGUAGES = { LANGUAGES = {

View File

@ -0,0 +1,103 @@
module Invidious::Videos
# Namespace for methods primarily relating to Transcripts
module Transcript
record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
kind = auto_generated ? "asr" : ""
object = {
"1:0:string" => video_id,
"2:base64" => {
"1:string" => kind,
"2:string" => language_code,
"3:string" => "",
},
"3:varint" => 1_i64,
"5:string" => "engagement-panel-searchable-transcript-search-panel",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"8:varint" => 1_i64,
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return params
end
def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
# Convert into array of TranscriptLine
lines = self.parse(initial_data)
# Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
vtt = String.build do |vtt|
vtt << <<-END_VTT
WEBVTT
Kind: captions
Language: #{target_language}
END_VTT
vtt << "\n\n"
lines.each do |line|
start_time = line.start_ms
end_time = line.end_ms
# start_time
vtt << start_time.hours.to_s.rjust(2, '0')
vtt << ':' << start_time.minutes.to_s.rjust(2, '0')
vtt << ':' << start_time.seconds.to_s.rjust(2, '0')
vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0')
vtt << " --> "
# end_time
vtt << end_time.hours.to_s.rjust(2, '0')
vtt << ':' << end_time.minutes.to_s.rjust(2, '0')
vtt << ':' << end_time.seconds.to_s.rjust(2, '0')
vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0')
vtt << "\n"
vtt << line.line
vtt << "\n"
vtt << "\n"
end
end
return vtt
end
private def self.parse(initial_data : Hash(String, JSON::Any))
body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
"content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
"initialSegments").as_a
lines = [] of TranscriptLine
body.each do |line|
# Transcript section headers. They are not apart of the captions and as such we can safely skip them.
if line.as_h.has_key?("transcriptSectionHeaderRenderer")
next
end
line = line["transcriptSegmentRenderer"]
start_ms = line["startMs"].as_s.to_i.millisecond
end_ms = line["endMs"].as_s.to_i.millisecond
text = extract_text(line["snippet"]) || ""
lines << TranscriptLine.new(start_ms, end_ms, text)
end
return lines
end
end
end

View File

@ -89,7 +89,7 @@
<label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label> <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
<% preferences.captions.each_with_index do |caption, index| %> <% preferences.captions.each_with_index do |caption, index| %>
<select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
<% Invidious::Videos::Caption::LANGUAGES.each do |option| %> <% Invidious::Videos::Captions::LANGUAGES.each do |option| %>
<option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
<% end %> <% end %>
</select> </select>

View File

@ -557,6 +557,30 @@ module YoutubeAPI
return self._post_json("/youtubei/v1/search", data, client_config) return self._post_json("/youtubei/v1/search", data, client_config)
end end
####################################################################
# get_transcript(params, client_config?)
#
# Requests the youtubei/v1/get_transcript endpoint with the required headers
# and POST data in order to get a JSON reply.
#
# The requested data is a specially encoded protobuf string that denotes the specific language requested.
#
# An optional ClientConfig parameter can be passed, too (see
# `struct ClientConfig` above for more details).
#
def get_transcript(
params : String,
client_config : ClientConfig | Nil = nil
) : Hash(String, JSON::Any)
data = {
"context" => self.make_context(client_config),
"params" => params,
}
return self._post_json("/youtubei/v1/get_transcript", data, client_config)
end
#################################################################### ####################################################################
# _post_json(endpoint, data, client_config?) # _post_json(endpoint, data, client_config?)
# #