mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-08-14.git
synced 2024-08-15 00:53:20 +00:00
Properly parse NewPipe imports
This commit is contained in:
parent
cdd916f51d
commit
fd7aa59e0f
2 changed files with 34 additions and 30 deletions
|
@ -16,7 +16,9 @@ dependencies:
|
||||||
commit: afd17fc
|
commit: afd17fc
|
||||||
pg:
|
pg:
|
||||||
github: will/crystal-pg
|
github: will/crystal-pg
|
||||||
|
sqlite3:
|
||||||
|
github: crystal-lang/crystal-sqlite3
|
||||||
|
|
||||||
crystal: 0.26.1
|
crystal: 0.27.0
|
||||||
|
|
||||||
license: AGPLv3
|
license: AGPLv3
|
||||||
|
|
|
@ -20,6 +20,7 @@ require "kemal"
|
||||||
require "openssl/hmac"
|
require "openssl/hmac"
|
||||||
require "option_parser"
|
require "option_parser"
|
||||||
require "pg"
|
require "pg"
|
||||||
|
require "sqlite3"
|
||||||
require "xml"
|
require "xml"
|
||||||
require "yaml"
|
require "yaml"
|
||||||
require "zip"
|
require "zip"
|
||||||
|
@ -1142,7 +1143,7 @@ get "/mark_unwatched" do |env|
|
||||||
|
|
||||||
if user
|
if user
|
||||||
user = user.as(User)
|
user = user.as(User)
|
||||||
PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE id = $2", id, user.id)
|
PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email)
|
||||||
end
|
end
|
||||||
|
|
||||||
if redirect
|
if redirect
|
||||||
|
@ -1320,12 +1321,13 @@ post "/data_control" do |env|
|
||||||
user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s }
|
user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s }
|
||||||
user.subscriptions.uniq!
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
user.subscriptions.each do |ucid|
|
user.subscriptions.select! do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
true
|
||||||
rescue ex
|
rescue ex
|
||||||
next
|
false
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -1349,12 +1351,13 @@ post "/data_control" do |env|
|
||||||
end
|
end
|
||||||
user.subscriptions.uniq!
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
user.subscriptions.each do |ucid|
|
user.subscriptions.select! do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
true
|
||||||
rescue ex
|
rescue ex
|
||||||
next
|
false
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -1365,12 +1368,13 @@ post "/data_control" do |env|
|
||||||
end
|
end
|
||||||
user.subscriptions.uniq!
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
user.subscriptions.each do |ucid|
|
user.subscriptions.select! do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
true
|
||||||
rescue ex
|
rescue ex
|
||||||
next
|
false
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -1396,34 +1400,32 @@ post "/data_control" do |env|
|
||||||
Zip::Reader.open(IO::Memory.new(body)) do |file|
|
Zip::Reader.open(IO::Memory.new(body)) do |file|
|
||||||
file.each_entry do |entry|
|
file.each_entry do |entry|
|
||||||
if entry.filename == "newpipe.db"
|
if entry.filename == "newpipe.db"
|
||||||
# We do this because the SQLite driver cannot parse a database from an IO
|
tempfile = File.tempfile(".db")
|
||||||
# Currently: channel URLs can **only** be subscriptions, and
|
File.write(tempfile.path, entry.io.gets_to_end)
|
||||||
# video URLs can **only** be watch history, so this works okay for now.
|
db = DB.open("sqlite3://" + tempfile.path)
|
||||||
|
|
||||||
db = entry.io.gets_to_end
|
user.watched += db.query_all("SELECT url FROM streams", as: String).map { |url| url.lchop("https://www.youtube.com/watch?v=") }
|
||||||
|
|
||||||
user.watched += db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/).map do |md|
|
|
||||||
md["id"]
|
|
||||||
end
|
|
||||||
user.watched.uniq!
|
user.watched.uniq!
|
||||||
|
|
||||||
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
|
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
|
||||||
|
|
||||||
user.subscriptions += db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/).map do |md|
|
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") }
|
||||||
md["ucid"]
|
|
||||||
end
|
|
||||||
user.subscriptions.uniq!
|
user.subscriptions.uniq!
|
||||||
|
|
||||||
user.subscriptions.each do |ucid|
|
user.subscriptions.select! do |ucid|
|
||||||
begin
|
begin
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(ucid, client, PG_DB, false, false)
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
true
|
||||||
rescue ex
|
rescue ex
|
||||||
next
|
false
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||||
|
|
||||||
|
db.close
|
||||||
|
tempfile.delete
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -1482,32 +1484,32 @@ get "/subscription_ajax" do |env|
|
||||||
|
|
||||||
# Update user
|
# Update user
|
||||||
if client.post(post_url, headers, post_req).status_code == 200
|
if client.post(post_url, headers, post_req).status_code == 200
|
||||||
sid = user.id
|
email = user.email
|
||||||
|
|
||||||
case action
|
case action
|
||||||
when .starts_with? "action_create"
|
when .starts_with? "action_create"
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", channel_id, sid)
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", channel_id, email)
|
||||||
when .starts_with? "action_remove"
|
when .starts_with? "action_remove"
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid)
|
PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
sid = user.id
|
email = user.email
|
||||||
|
|
||||||
case action
|
case action
|
||||||
when .starts_with? "action_create"
|
when .starts_with? "action_create"
|
||||||
if !user.subscriptions.includes? channel_id
|
if !user.subscriptions.includes? channel_id
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", channel_id, sid)
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", channel_id, email)
|
||||||
|
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
get_channel(channel_id, client, PG_DB, false, false)
|
get_channel(channel_id, client, PG_DB, false, false)
|
||||||
end
|
end
|
||||||
when .starts_with? "action_remove"
|
when .starts_with? "action_remove"
|
||||||
PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid)
|
PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE email = $2", channel_id, email)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE id = $1", [sid], as: Int64)
|
count_text = PG_DB.query_one?("SELECT cardinality(subscriptions) FROM users WHERE email = $1", email, as: Int32)
|
||||||
count_text ||= 0
|
count_text ||= 0
|
||||||
count_text = "#{number_with_separator(count_text)} subscriptions"
|
count_text = "#{number_with_separator(count_text)} subscriptions"
|
||||||
end
|
end
|
||||||
|
@ -1742,8 +1744,8 @@ get "/feed/subscriptions" do |env|
|
||||||
videos = videos[0..max_results]
|
videos = videos[0..max_results]
|
||||||
end
|
end
|
||||||
|
|
||||||
PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE id = $3", [] of String, Time.now,
|
PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.now,
|
||||||
user.id)
|
user.email)
|
||||||
user.notifications = [] of String
|
user.notifications = [] of String
|
||||||
env.set "user", user
|
env.set "user", user
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue