Add support for subscribing to channels via PubSubHubbub

src/invidious/channels.cr

@@ -1,9 +1,10 @@
 class InvidiousChannel
   add_mapping({
-    id:      String,
-    author:  String,
-    updated: Time,
-    deleted: Bool,
+    id:         String,
+    author:     String,
+    updated:    Time,
+    deleted:    Bool,
+    subscribed: {type: Bool, default: false},
   })
 end
 
@@ -15,10 +16,7 @@ class ChannelVideo
     updated:        Time,
     ucid:           String,
     author:         String,
-    length_seconds: {
-      type:    Int32,
-      default: 0,
-    },
+    length_seconds: {type: Int32, default: 0},
   })
 end
 
@@ -188,11 +186,29 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
     db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
   end
 
-  channel = InvidiousChannel.new(ucid, author, Time.now, false)
+  channel = InvidiousChannel.new(ucid, author, Time.now, false, false)
 
   return channel
 end
 
+def subscribe_pubsub(ucid, key, config)
+  client = make_client(PUBSUB_URL)
+  time = Time.now.to_unix.to_s
+
+  host_url = make_host_url(Kemal.config.ssl || config.https_only, config.domain)
+
+  body = {
+    "hub.callback"     => "#{host_url}/feed/webhook",
+    "hub.topic"        => "https://www.youtube.com/feeds/videos.xml?channel_id=#{ucid}",
+    "hub.verify"       => "async",
+    "hub.mode"         => "subscribe",
+    "hub.verify_token" => "#{time}:#{OpenSSL::HMAC.hexdigest(:sha1, key, time)}",
+    "hub.secret"       => key.to_s,
+  }
+
+  return client.post("/subscribe", form: body)
+end
+
 def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
   client = make_client(YT_URL)
 

src/invidious/helpers/helpers.cr

@@ -11,10 +11,11 @@ user: String,
       port: Int32,
       dbname: String,
     ),
-    full_refresh:         Bool,    # Used for crawling channels: threads should check all videos uploaded by a channel
-    https_only:           Bool?,   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
-    hmac_key:             String?, # HMAC signing key for CSRF tokens
-    domain:               String?, # Domain to be used for links to resources on the site where an absolute URL is required
+    full_refresh:         Bool,                         # Used for crawling channels: threads should check all videos uploaded by a channel
+    https_only:           Bool?,                        # Used to tell Invidious it is behind a proxy, so links to resources should be https://
+    hmac_key:             String?,                      # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
+    domain:               String?,                      # Domain to be used for links to resources on the site where an absolute URL is required
+    use_pubsub_feeds:     {type: Bool, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
     default_home:         {type: String, default: "Top"},
     feed_menu:            {type: Array(String), default: ["Popular", "Top", "Trending"]},
     top_enabled:          {type: Bool, default: true},

src/invidious/jobs.cr

@@ -153,6 +153,26 @@ def refresh_feeds(db, logger, max_threads = 1)
   max_channel.send(max_threads)
 end
 
+def subscribe_to_feeds(db, logger, key, config)
+  if config.use_pubsub_feeds
+    spawn do
+      loop do
+        db.query_all("SELECT ucid FROM channels WHERE subscribed = false") do |rs|
+          ucid = rs.read(String)
+          response = subscribe_pubsub(ucid, key, config)
+
+          if response.status_code >= 400
+            logger.write("#{ucid} : #{response.body}\n")
+          end
+        end
+
+        sleep 1.minute
+        Fiber.yield
+      end
+    end
+  end
+end
+
 def pull_top_videos(config, db)
   loop do
     begin