Merge pull request #2545 from bbielsa/csv-subscriptions-import

Add CSV Subscriptions Import
This commit is contained in:
Samantaz Fox 2022-01-16 15:11:37 +01:00 committed by GitHub
commit 6fab5d0554
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 93 additions and 3 deletions

View file

@ -0,0 +1,51 @@
require "spectator"
require "../../../src/invidious/user/imports"
Spectator.configure do |config|
config.fail_blank
config.randomize
end
def csv_sample
return <<-CSV
Kanal-ID,Kanal-URL,Kanaltitel
UC0hHW5Y08ggq-9kbrGgWj0A,http://www.youtube.com/channel/UC0hHW5Y08ggq-9kbrGgWj0A,Matias Marolla
UC0vBXGSyV14uvJ4hECDOl0Q,http://www.youtube.com/channel/UC0vBXGSyV14uvJ4hECDOl0Q,Techquickie
UC1sELGmy5jp5fQUugmuYlXQ,http://www.youtube.com/channel/UC1sELGmy5jp5fQUugmuYlXQ,Minecraft
UC9kFnwdCRrX7oTjqKd6-tiQ,http://www.youtube.com/channel/UC9kFnwdCRrX7oTjqKd6-tiQ,LUMOX - Topic
UCBa659QWEk1AI4Tg--mrJ2A,http://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A,Tom Scott
UCGu6_XQ64rXPR6nuitMQE_A,http://www.youtube.com/channel/UCGu6_XQ64rXPR6nuitMQE_A,Callcenter Fun
UCGwu0nbY2wSkW8N-cghnLpA,http://www.youtube.com/channel/UCGwu0nbY2wSkW8N-cghnLpA,Jaiden Animations
UCQ0OvZ54pCFZwsKxbltg_tg,http://www.youtube.com/channel/UCQ0OvZ54pCFZwsKxbltg_tg,Methos
UCRE6itj4Jte4manQEu3Y7OA,http://www.youtube.com/channel/UCRE6itj4Jte4manQEu3Y7OA,Chipflake
UCRLc6zsv_d0OEBO8OOkz-DA,http://www.youtube.com/channel/UCRLc6zsv_d0OEBO8OOkz-DA,Kegy
UCSl5Uxu2LyaoAoMMGp6oTJA,http://www.youtube.com/channel/UCSl5Uxu2LyaoAoMMGp6oTJA,Atomic Shrimp
UCXuqSBlHAE6Xw-yeJA0Tunw,http://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw,Linus Tech Tips
UCZ5XnGb-3t7jCkXdawN2tkA,http://www.youtube.com/channel/UCZ5XnGb-3t7jCkXdawN2tkA,Discord
CSV
end
Spectator.describe "Invidious::User::Imports" do
it "imports CSV" do
subscriptions = parse_subscription_export_csv(csv_sample)
expect(subscriptions).to be_an(Array(String))
expect(subscriptions.size).to eq(13)
expect(subscriptions).to contain_exactly(
"UC0hHW5Y08ggq-9kbrGgWj0A",
"UC0vBXGSyV14uvJ4hECDOl0Q",
"UC1sELGmy5jp5fQUugmuYlXQ",
"UC9kFnwdCRrX7oTjqKd6-tiQ",
"UCBa659QWEk1AI4Tg--mrJ2A",
"UCGu6_XQ64rXPR6nuitMQE_A",
"UCGwu0nbY2wSkW8N-cghnLpA",
"UCQ0OvZ54pCFZwsKxbltg_tg",
"UCRE6itj4Jte4manQEu3Y7OA",
"UCRLc6zsv_d0OEBO8OOkz-DA",
"UCSl5Uxu2LyaoAoMMGp6oTJA",
"UCXuqSBlHAE6Xw-yeJA0Tunw",
"UCZ5XnGb-3t7jCkXdawN2tkA",
).in_order
end
end

View file

@ -746,6 +746,8 @@ post "/data_control" do |env|
HTTP::FormData.parse(env.request) do |part| HTTP::FormData.parse(env.request) do |part|
body = part.body.gets_to_end body = part.body.gets_to_end
type = part.headers["Content-Type"]
next if body.empty? next if body.empty?
# TODO: Unify into single import based on content-type # TODO: Unify into single import based on content-type
@ -816,19 +818,29 @@ post "/data_control" do |env|
end end
end end
when "import_youtube" when "import_youtube"
if body[0..4] == "<opml" filename = part.filename || ""
extension = filename.split(".").last
if extension == "xml" || type == "application/xml" || type == "text/xml"
subscriptions = XML.parse(body) subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end end
else elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body) subscriptions = JSON.parse(body)
user.subscriptions += subscriptions.as_a.compact_map do |entry| user.subscriptions += subscriptions.as_a.compact_map do |entry|
entry["snippet"]["resourceId"]["channelId"].as_s entry["snippet"]["resourceId"]["channelId"].as_s
end end
elsif extension == "csv" || type == "text/csv"
subscriptions = parse_subscription_export_csv(body)
user.subscriptions += subscriptions
else
halt(env, status_code: 415,
response: error_template(415, "Invalid subscription file uploaded")
)
end end
user.subscriptions.uniq!
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions, false, false) user.subscriptions = get_batch_channels(user.subscriptions, false, false)
Invidious::Database::Users.update_subscriptions(user) Invidious::Database::Users.update_subscriptions(user)

View file

@ -0,0 +1,27 @@
require "csv"
def parse_subscription_export_csv(csv_content : String)
rows = CSV.new(csv_content, headers: true)
subscriptions = Array(String).new
# Counter to limit the amount of imports.
# This is intended to prevent DoS.
row_counter = 0
rows.each do |row|
# Limit to 1200
row_counter += 1
break if row_counter > 1_200
# Channel ID is the first column in the csv export we can't use the header
# name, because the header name is localized depending on the
# language the user has set on their account
channel_id = row[0].strip
next if channel_id.empty?
subscriptions << channel_id
end
return subscriptions
end