From 6d8caaee654111009b8f47d20a380e4033a92d10 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 8 Aug 2018 20:12:17 -0500 Subject: [PATCH] Add option to pull all videos --- config/config.yml | 3 ++- src/invidious.cr | 2 +- src/invidious/helpers/helpers.cr | 7 ++++--- src/invidious/jobs.cr | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/config/config.yml b/config/config.yml index a2cc5d9c..94c2766a 100644 --- a/config/config.yml +++ b/config/config.yml @@ -6,4 +6,5 @@ db: password: kemal host: localhost port: 5432 - dbname: invidious \ No newline at end of file + dbname: invidious +full_refresh: false \ No newline at end of file diff --git a/src/invidious.cr b/src/invidious.cr index f1e8c691..a7e2de53 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -83,7 +83,7 @@ crawl_threads.times do end end -refresh_channels(PG_DB, channel_threads) +refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh) video_threads.times do |i| spawn do diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index fcc191f6..0d115e68 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -10,9 +10,10 @@ class Config port: Int32, dbname: String, ), - dl_api_key: String?, - https_only: Bool?, - hmac_key: String?, + dl_api_key: String?, + https_only: Bool?, + hmac_key: String?, + full_refresh: Bool, }) end diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index 7d45d7ce..b74bcaf2 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -44,7 +44,7 @@ def crawl_videos(db) end end -def refresh_channels(db, max_threads = 1) +def refresh_channels(db, max_threads = 1, full_refresh = false) max_channel = Channel(Int32).new spawn do @@ -67,7 +67,7 @@ def refresh_channels(db, max_threads = 1) spawn do begin client = make_client(YT_URL) - channel = fetch_channel(id, client, db, false) + channel = fetch_channel(id, client, db, full_refresh) db.exec("UPDATE channels SET updated = $1 WHERE id = $2", Time.now, id) rescue ex