iv-org-mirror/invidious
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious.git synced 2024-08-15 00:53:41 +00:00
Code Issues Projects Releases Packages Wiki Activity

API: Remove the fields parameter (#4276)

Browse source 
Multiple users have reported that the fields parameter is slowing down API
response times significantly. As most API endpoints are already optimized to
make as few requests as possible to Youtube, there is no point in limiting the
output. Furthermore, the added processing might be part of the broader memory
leak problem (See 1438).

In addition, the small increase in data output is not much of an issue compared
to the huge video proxy that lies next to this API.

No related issue tracked
This commit is contained in:
Samantaz Fox 2024-02-12 22:10:45 +01:00
commit 1f51255f2f
No known key found for this signature in database
GPG key ID: F42821059186176E
5 changed files with 2 additions and 346 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
assets/js/notifications.js
View file

@ -10,7 +10,7 @@ var notifications, delivered;
var notifications_mock = { close: function () { } }; var notifications_mock = { close: function () { } };
function get_subscriptions() { function get_subscriptions() {
helpers.xhr('GET', '/api/v1/auth/subscriptions?fields=authorId', { helpers.xhr('GET', '/api/v1/auth/subscriptions', {
retries: 5, retries: 5,
entity_name: 'subscriptions' entity_name: 'subscriptions'
}, { }, {
@ -22,7 +22,7 @@ function create_notification_stream(subscriptions) {
// sse.js can't be replaced to EventSource in place as it lack support of payload and headers // sse.js can't be replaced to EventSource in place as it lack support of payload and headers
// see https://developer.mozilla.org/en-US/docs/Web/API/EventSource/EventSource // see https://developer.mozilla.org/en-US/docs/Web/API/EventSource/EventSource
notifications = new SSE( notifications = new SSE(
'/api/v1/auth/notifications?fields=videoId,title,author,authorId,publishedText,published,authorThumbnails,liveNow', { '/api/v1/auth/notifications', {
withCredentials: true, withCredentials: true,
payload: 'topics=' + subscriptions.map(function (subscription) { return subscription.authorId; }).join(','), payload: 'topics=' + subscriptions.map(function (subscription) { return subscription.authorId; }).join(','),
headers: { 'Content-Type': 'application/x-www-form-urlencoded' } headers: { 'Content-Type': 'application/x-www-form-urlencoded' }

1
src/invidious.cr
View file

@ -217,7 +217,6 @@ public_folder "assets"
Kemal.config.powered_by_header = false Kemal.config.powered_by_header = false
add_handler FilteredCompressHandler.new add_handler FilteredCompressHandler.new
add_handler APIHandler.new
add_handler AuthHandler.new add_handler AuthHandler.new
add_handler DenyFrame.new add_handler DenyFrame.new
add_context_storage_type(Array(String)) add_context_storage_type(Array(String))

68
src/invidious/helpers/handlers.cr
View file

@ -134,74 +134,6 @@ class AuthHandler < Kemal::Handler
end end
end end
class APIHandler < Kemal::Handler
{% for method in %w(GET POST PUT HEAD DELETE PATCH OPTIONS) %}
only ["/api/v1/*"], {{method}}
{% end %}
exclude ["/api/v1/auth/notifications"], "GET"
exclude ["/api/v1/auth/notifications"], "POST"
def call(env)
return call_next env unless only_match? env
env.response.headers["Access-Control-Allow-Origin"] = "*"
# Since /api/v1/notifications is an event-stream, we don't want
# to wrap the response
return call_next env if exclude_match? env
# Here we swap out the socket IO so we can modify the response as needed
output = env.response.output
env.response.output = IO::Memory.new
begin
call_next env
env.response.output.rewind
if env.response.output.as(IO::Memory).size != 0 &&
env.response.headers.includes_word?("Content-Type", "application/json")
response = JSON.parse(env.response.output)
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
else
response = env.response.output.gets_to_end
end
rescue ex
env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
env.response.status_code = 500
if env.response.headers.includes_word?("Content-Type", "application/json")
response = {"error" => ex.message || "Unspecified error"}
if env.params.query["pretty"]?.try &.== "1"
response = response.to_pretty_json
else
response = response.to_json
end
end
ensure
env.response.output = output
env.response.print response
env.response.flush
end
end
end
class DenyFrame < Kemal::Handler class DenyFrame < Kemal::Handler
exclude ["/embed/*"] exclude ["/embed/*"]

27
src/invidious/helpers/helpers.cr
View file

@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel)
video.published = published video.published = published
response = JSON.parse(video.to_json(locale, nil)) response = JSON.parse(video.to_json(locale, nil))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}" env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}" env.response.puts "data: #{response.to_json}"
env.response.puts env.response.puts
@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel)
Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
response = JSON.parse(video.to_json(locale)) response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}" env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}" env.response.puts "data: #{response.to_json}"
env.response.puts env.response.puts
@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel)
video.published = Time.unix(published) video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale, nil)) response = JSON.parse(video.to_json(locale, nil))
if fields_text = env.params.query["fields"]?
begin
JSONFilter.filter(response, fields_text)
rescue ex
env.response.status_code = 400
response = {"error" => ex.message}
end
end
env.response.puts "id: #{id}" env.response.puts "id: #{id}"
env.response.puts "data: #{response.to_json}" env.response.puts "data: #{response.to_json}"
env.response.puts env.response.puts

248
src/invidious/helpers/json_filter.cr
View file

@ -1,248 +0,0 @@
module JSONFilter
alias BracketIndex = Hash(Int64, Int64)
alias GroupedFieldsValue = String | Array(GroupedFieldsValue)
alias GroupedFieldsList = Array(GroupedFieldsValue)
class FieldsParser
class ParseError < Exception
end
# Returns the `Regex` pattern used to match nest groups
def self.nest_group_pattern : Regex
# uses a '.' character to match json keys as they are allowed
# to contain any unicode codepoint
/(?:|,)(?<groupname>[^,\n]*?)\(/
end
# Returns the `Regex` pattern used to check if there are any empty nest groups
def self.unnamed_nest_group_pattern : Regex
/^\(|\(\(|\/\(/
end
def self.parse_fields(fields_text : String, &) : Nil
if fields_text.empty?
raise FieldsParser::ParseError.new "Fields is empty"
end
opening_bracket_count = fields_text.count('(')
closing_bracket_count = fields_text.count(')')
if opening_bracket_count != closing_bracket_count
bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
elsif match_result = unnamed_nest_group_pattern.match(fields_text)
raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
end
# first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
parse_single_nests(fields_text) { |nest_list| yield nest_list }
# next, handle nest groups: items(id, etag, etc)
parse_nest_groups(fields_text) { |nest_list| yield nest_list }
end
def self.parse_single_nests(fields_text : String, &) : Nil
single_nests = remove_nest_groups(fields_text)
if !single_nests.empty?
property_nests = single_nests.split(',')
property_nests.each do |nest|
nest_list = nest.split('/')
if nest_list.includes? ""
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
end
yield nest_list
end
# else
# raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
end
end
def self.parse_nest_groups(fields_text : String, &) : Nil
nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
bracket_pairs = get_bracket_pairs(fields_text, true)
text_index = 0
regex_index = 0
while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
raw_match = regex_result[0]
group_name = regex_result["groupname"]
text_index = regex_result.begin
regex_index = regex_result.end
if text_index.nil? || regex_index.nil?
raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
end
offset = raw_match.starts_with?(',') ? 1 : 0
opening_bracket_index = (text_index + group_name.size) + offset
closing_bracket_index = bracket_pairs[opening_bracket_index]
content_start = opening_bracket_index + 1
content = fields_text[content_start...closing_bracket_index]
if content.empty?
raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
else
content = remove_nest_groups(content)
end
while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
if nest_stack.size
nest_stack.pop
end
end
group_name.split('/').each do |name|
nest_stack.push({
group_name: name,
closing_bracket_index: closing_bracket_index,
})
end
if !content.empty?
properties = content.split(',')
properties.each do |prop|
nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
if !prop.empty?
if prop.includes?('/')
parse_single_nests(prop) { |list| nest_list += list }
else
nest_list.push prop
end
else
raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
end
yield nest_list
end
end
end
end
def self.remove_nest_groups(text : String) : String
content_bracket_pairs = get_bracket_pairs(text, false)
content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
closing_bracket = content_bracket_pairs[opening_bracket]
last_comma = text.rindex(',', opening_bracket) || 0
text = text[0...last_comma] + text[closing_bracket + 1...text.size]
end
return text.starts_with?(',') ? text[1...text.size] : text
end
def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
istart = [] of Int64
bracket_index = BracketIndex.new
text.each_char_with_index do |char, index|
if char == '('
istart.push(index.to_i64)
end
if char == ')'
begin
opening = istart.pop
if recursive || (!recursive && istart.size == 0)
bracket_index[opening] = index.to_i64
end
rescue
raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
end
end
end
if istart.size != 0
idx = istart.pop
raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
end
return bracket_index
end
end
class FieldsGrouper
alias SkeletonValue = Hash(String, SkeletonValue)
def self.create_json_skeleton(fields_text : String) : SkeletonValue
root_hash = {} of String => SkeletonValue
FieldsParser.parse_fields(fields_text) do |nest_list|
current_item = root_hash
nest_list.each do |key|
if current_item[key]?
current_item = current_item[key]
else
current_item[key] = {} of String => SkeletonValue
current_item = current_item[key]
end
end
end
root_hash
end
def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
grouped_fields_list = GroupedFieldsList.new
json_skeleton.each do |key, value|
grouped_fields_list.push key
nested_keys = create_grouped_fields_list(value)
grouped_fields_list.push nested_keys unless nested_keys.empty?
end
return grouped_fields_list
end
end
class FilterError < Exception
end
def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
skeleton = FieldsGrouper.create_json_skeleton(fields_text)
grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
filter(item, grouped_fields_list, in_place)
end
def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
item = item.clone unless in_place
if !item.as_h? && !item.as_a?
raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
end
top_level_keys = Array(String).new
grouped_fields_list.each do |value|
if value.is_a? String
top_level_keys.push value
elsif value.is_a? Array
if !top_level_keys.empty?
key_to_filter = top_level_keys.last
if item.as_h?
filter(item[key_to_filter], value, in_place: true)
elsif item.as_a?
item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
end
else
raise FilterError.new "Tried to filter while top level keys list is empty"
end
end
end
if item.as_h?
item.as_h.select! top_level_keys
elsif item.as_a?
item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
end
item
end
end