iv-org-mirror/invidious-copy-2022-03-16
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2022-03-16.git synced 2024-08-15 00:53:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1608 from saltycrys/add-subscription-traces

Browse source 
Add Subscription Traces
This commit is contained in:
Perflyst 2020-12-31 11:30:04 +01:00 • committed by GitHub
commit c89632d2a8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 72 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/invidious/jobs/refresh_channels_job.cr
View file

@ -7,9 +7,9 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
end
def begin
max_threads = config.channel_threads
lim_threads = max_threads
active_threads = 0
max_fibers = config.channel_threads
lim_fibers = max_fibers
active_fibers = 0
active_channel = Channel(Bool).new
backoff = 1.seconds
@ -19,27 +19,32 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
rs.each do
id = rs.read(String)
if active_threads >= lim_threads
if active_fibers >= lim_fibers
logger.trace("RefreshChannelsJob: Fiber limit reached, waiting...")
if active_channel.receive
active_threads -= 1
logger.trace("RefreshChannelsJob: Fiber limit ok, continuing")
active_fibers -= 1
end
end
active_threads += 1
logger.trace("RefreshChannelsJob: #{id} : Spawning fiber")
active_fibers += 1
spawn do
begin
logger.trace("RefreshChannelsJob: Fetching channel #{id}")
channel = fetch_channel(id, db, config.full_refresh)
logger.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, db, logger, config.full_refresh)
lim_threads = max_threads
lim_fibers = max_fibers
logger.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
rescue ex
logger.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel"
db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
else
lim_threads = 1
logger.error("RefreshChannelsJob: #{id} : backing off for #{backoff}s")
lim_fibers = 1
logger.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s")
sleep backoff
if backoff < 1.days
backoff += backoff
@ -47,13 +52,15 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
backoff = 1.days
end
end
ensure
logger.trace("RefreshChannelsJob: #{id} fiber : Done")
active_channel.send(true)
end
active_channel.send(true)
end
end
end
logger.debug("RefreshChannelsJob: Done, sleeping for one minute")
sleep 1.minute
Fiber.yield
end

10
src/invidious/jobs/refresh_feeds_job.cr
View file

@ -7,8 +7,8 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
end
def begin
max_threads = config.feed_threads
active_threads = 0
max_fibers = config.feed_threads
active_fibers = 0
active_channel = Channel(Bool).new
loop do
@ -17,13 +17,13 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
email = rs.read(String)
view_name = "subscriptions_#{sha256(email)}"
if active_threads >= max_threads
if active_fibers >= max_fibers
if active_channel.receive
active_threads -= 1
active_fibers -= 1
end
end
active_threads += 1
active_fibers += 1
spawn do
begin
# Drop outdated views

12
src/invidious/jobs/subscribe_to_feeds_job.cr
View file

@ -8,12 +8,12 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
def begin
max_threads = 1
max_fibers = 1
if config.use_pubsub_feeds.is_a?(Int32)
max_threads = config.use_pubsub_feeds.as(Int32)
max_fibers = config.use_pubsub_feeds.as(Int32)
end
active_threads = 0
active_fibers = 0
active_channel = Channel(Bool).new
loop do
@ -21,13 +21,13 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
rs.each do
ucid = rs.read(String)
if active_threads >= max_threads.as(Int32)
if active_fibers >= max_fibers.as(Int32)
if active_channel.receive
active_threads -= 1
active_fibers -= 1
end
end
active_threads += 1
active_fibers += 1
spawn do
begin