Merge pull request #1389 from vhuynh3000/decrypt_on_demand

add config to decrypt on demand instead of polling
This commit is contained in:
Perflyst 2021-01-05 19:32:22 +01:00 committed by GitHub
commit 2c24bf3222
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 74 additions and 51 deletions

View file

@ -168,7 +168,11 @@ end
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config)
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config)
Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY) Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY)
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new
DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling)
if config.decrypt_polling
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new(logger)
end
if config.statistics_enabled if config.statistics_enabled
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE) Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE)
@ -191,8 +195,6 @@ def popular_videos
Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
end end
DECRYPT_FUNCTION = Invidious::Jobs::UpdateDecryptFunctionJob::DECRYPT_FUNCTION
before_all do |env| before_all do |env|
preferences = begin preferences = begin
Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")

View file

@ -67,6 +67,7 @@ class Config
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
property feed_threads : Int32 # Number of threads to use for updating feeds property feed_threads : Int32 # Number of threads to use for updating feeds
property db : DBConfig # Database configuration property db : DBConfig # Database configuration
property decrypt_polling : Bool = true # Use polling to keep decryption function up to date
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions

View file

@ -1,6 +1,17 @@
alias SigProc = Proc(Array(String), Int32, Array(String)) alias SigProc = Proc(Array(String), Int32, Array(String))
def fetch_decrypt_function(id = "CvFH_6DNRCY") struct DecryptFunction
@decrypt_function = [] of {SigProc, Int32}
@decrypt_time = Time.monotonic
def initialize(@use_polling = true)
end
def update_decrypt_function
@decrypt_function = fetch_decrypt_function
end
private def fetch_decrypt_function(id = "CvFH_6DNRCY")
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
player = YT_POOL.client &.get(url).body player = YT_POOL.client &.get(url).body
@ -38,16 +49,25 @@ def fetch_decrypt_function(id = "CvFH_6DNRCY")
end end
return decrypt_function return decrypt_function
end end
def decrypt_signature(fmt : Hash(String, JSON::Any)) def decrypt_signature(fmt : Hash(String, JSON::Any))
return "" if !fmt["s"]? || !fmt["sp"]? return "" if !fmt["s"]? || !fmt["sp"]?
sp = fmt["sp"].as_s sp = fmt["sp"].as_s
sig = fmt["s"].as_s.split("") sig = fmt["s"].as_s.split("")
DECRYPT_FUNCTION.each do |proc, value| if !@use_polling
now = Time.monotonic
if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
@decrypt_function = fetch_decrypt_function
@decrypt_time = Time.monotonic
end
end
@decrypt_function.each do |proc, value|
sig = proc.call(sig, value) sig = proc.call(sig, value)
end end
return "&#{sp}=#{sig.join("")}" return "&#{sp}=#{sig.join("")}"
end
end end

View file

@ -1,15 +1,15 @@
class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
DECRYPT_FUNCTION = [] of {SigProc, Int32} private getter logger : Invidious::LogHandler
def initialize(@logger)
end
def begin def begin
loop do loop do
begin begin
decrypt_function = fetch_decrypt_function DECRYPT_FUNCTION.update_decrypt_function
DECRYPT_FUNCTION.clear
decrypt_function.each { |df| DECRYPT_FUNCTION << df }
rescue ex rescue ex
# TODO: Log error logger.error("UpdateDecryptFunctionJob : #{ex.message}")
next
ensure ensure
sleep 1.minute sleep 1.minute
Fiber.yield Fiber.yield

View file

@ -580,7 +580,7 @@ struct Video
s.each do |k, v| s.each do |k, v|
fmt[k] = JSON::Any.new(v) fmt[k] = JSON::Any.new(v)
end end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
@ -599,7 +599,7 @@ struct Video
s.each do |k, v| s.each do |k, v|
fmt[k] = JSON::Any.new(v) fmt[k] = JSON::Any.new(v)
end end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")