iv-org-mirror/invidious-copy-2022-03-16
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2022-03-16.git synced 2024-08-15 00:53:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add multithreading to pubsub job

Browse source
This commit is contained in:
Omar Roth 2019-04-04 07:49:53 -05:00
parent 31312747e9
commit 305d636217
3 changed files with 43 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/invidious/helpers/helpers.cr
View file

@ -81,11 +81,11 @@ user: String,
port: Int32,
dbname: String,
),
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
default_home: {type: String, default: "Top"},
feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending", "Subscriptions"]},
top_enabled: {type: Bool, default: true},

38
src/invidious/jobs.cr
View file

@ -104,15 +104,43 @@ end
def subscribe_to_feeds(db, logger, key, config)
if config.use_pubsub_feeds
case config.use_pubsub_feeds
when Bool
max_threads = config.use_pubsub_feeds.as(Bool).to_unsafe
when Int32
max_threads = config.use_pubsub_feeds.as(Int32)
end
max_channel = Channel(Int32).new
spawn do
max_threads = max_channel.receive
active_threads = 0
active_channel = Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > '4 days' OR subscribed IS NULL") do |rs|
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
rs.each do
ucid = rs.read(String)
response = subscribe_pubsub(ucid, key, config)
if response.status_code >= 400
logger.write("#{ucid} : #{response.body}\n")
if active_threads >= max_threads.as(Int32)
if active_channel.receive
active_threads -= 1
end
end
active_threads += 1
spawn do
begin
response = subscribe_pubsub(ucid, key, config)
if response.status_code >= 400
logger.write("#{ucid} : #{response.body}\n")
end
rescue ex
end
active_channel.send(true)
end
end
end
@ -120,6 +148,8 @@ def subscribe_to_feeds(db, logger, key, config)
sleep 1.minute
end
end
max_channel.send(max_threads.as(Int32))
end
end