Add support for subscribing to channels via PubSubHubbub

This commit is contained in:
Omar Roth 2019-03-03 19:18:23 -06:00
parent 17cf0772fb
commit 64cfd2296c
5 changed files with 110 additions and 14 deletions

View file

@ -0,0 +1,4 @@
#!/bin/sh
psql invidious -c "ALTER TABLE channels ADD COLUMN subscribed bool;"
psql invidious -c "UPDATE channels SET subscribed = false;"

View file

@ -28,7 +28,7 @@ require "./invidious/helpers/*"
require "./invidious/*" require "./invidious/*"
CONFIG = Config.from_yaml(File.read("config/config.yml")) CONFIG = Config.from_yaml(File.read("config/config.yml"))
HMAC_KEY = CONFIG.hmac_key || Random::Secure.random_bytes(32) HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
config = CONFIG config = CONFIG
logger = Invidious::LogHandler.new logger = Invidious::LogHandler.new
@ -88,6 +88,7 @@ PG_DB = DB.open PG_URL
YT_URL = URI.parse("https://www.youtube.com") YT_URL = URI.parse("https://www.youtube.com")
REDDIT_URL = URI.parse("https://www.reddit.com") REDDIT_URL = URI.parse("https://www.reddit.com")
LOGIN_URL = URI.parse("https://accounts.google.com") LOGIN_URL = URI.parse("https://accounts.google.com")
PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com/omarroth@hotmail.com.json") TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com/omarroth@hotmail.com.json")
CURRENT_COMMIT = `git rev-list HEAD --max-count=1 --abbrev-commit`.strip CURRENT_COMMIT = `git rev-list HEAD --max-count=1 --abbrev-commit`.strip
CURRENT_VERSION = `git describe --tags $(git rev-list --tags --max-count=1)`.strip CURRENT_VERSION = `git describe --tags $(git rev-list --tags --max-count=1)`.strip
@ -115,6 +116,8 @@ refresh_channels(PG_DB, logger, config.channel_threads, config.full_refresh)
refresh_feeds(PG_DB, logger, config.feed_threads) refresh_feeds(PG_DB, logger, config.feed_threads)
subscribe_to_feeds(PG_DB, logger, HMAC_KEY, config)
config.video_threads.times do |i| config.video_threads.times do |i|
spawn do spawn do
refresh_videos(PG_DB, logger) refresh_videos(PG_DB, logger)
@ -2314,6 +2317,58 @@ get "/feed/playlist/:plid" do |env|
document document
end end
# Add support for subscribing to channels via PubSubHubbub
get "/feed/webhook" do |env|
mode = env.params.query["hub.mode"]
topic = env.params.query["hub.topic"]
challenge = env.params.query["hub.challenge"]
lease_seconds = env.params.query["hub.lease_seconds"]
verify_token = env.params.query["hub.verify_token"]
time, signature = verify_token.split(":")
if Time.now.to_unix - time.to_i > 600
halt env, status_code: 400
end
if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, time) != signature
halt env, status_code: 400
end
ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]
PG_DB.exec("UPDATE channels SET subscribed = true WHERE ucid = $1", ucid)
halt env, status_code: 200, response: challenge
end
post "/feed/webhook" do |env|
body = env.request.body.not_nil!.gets_to_end
signature = env.request.headers["X-Hub-Signature"].lchop("sha1=")
if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body)
halt env, status_code: 200
end
rss = XML.parse_html(body)
rss.xpath_nodes("//feed/entry").each do |entry|
id = entry.xpath_node("videoid").not_nil!.content
video = get_video(id, PG_DB, proxies)
video = ChannelVideo.new(id, video.title, video.published, Time.now, video.ucid, video.author, video.length_seconds)
PG_DB.exec("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, video.ucid)
video_array = video.to_a
args = arg_array(video_array)
PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7", video_array)
end
end
# Channels # Channels
# YouTube appears to let users set a "brand" URL that # YouTube appears to let users set a "brand" URL that

View file

@ -1,9 +1,10 @@
class InvidiousChannel class InvidiousChannel
add_mapping({ add_mapping({
id: String, id: String,
author: String, author: String,
updated: Time, updated: Time,
deleted: Bool, deleted: Bool,
subscribed: {type: Bool, default: false},
}) })
end end
@ -15,10 +16,7 @@ class ChannelVideo
updated: Time, updated: Time,
ucid: String, ucid: String,
author: String, author: String,
length_seconds: { length_seconds: {type: Int32, default: 0},
type: Int32,
default: 0,
},
}) })
end end
@ -188,11 +186,29 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid) db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
end end
channel = InvidiousChannel.new(ucid, author, Time.now, false) channel = InvidiousChannel.new(ucid, author, Time.now, false, false)
return channel return channel
end end
def subscribe_pubsub(ucid, key, config)
client = make_client(PUBSUB_URL)
time = Time.now.to_unix.to_s
host_url = make_host_url(Kemal.config.ssl || config.https_only, config.domain)
body = {
"hub.callback" => "#{host_url}/feed/webhook",
"hub.topic" => "https://www.youtube.com/feeds/videos.xml?channel_id=#{ucid}",
"hub.verify" => "async",
"hub.mode" => "subscribe",
"hub.verify_token" => "#{time}:#{OpenSSL::HMAC.hexdigest(:sha1, key, time)}",
"hub.secret" => key.to_s,
}
return client.post("/subscribe", form: body)
end
def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
client = make_client(YT_URL) client = make_client(YT_URL)

View file

@ -11,10 +11,11 @@ user: String,
port: Int32, port: Int32,
dbname: String, dbname: String,
), ),
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https:// https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds: {type: Bool, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
default_home: {type: String, default: "Top"}, default_home: {type: String, default: "Top"},
feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending"]}, feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending"]},
top_enabled: {type: Bool, default: true}, top_enabled: {type: Bool, default: true},

View file

@ -153,6 +153,26 @@ def refresh_feeds(db, logger, max_threads = 1)
max_channel.send(max_threads) max_channel.send(max_threads)
end end
def subscribe_to_feeds(db, logger, key, config)
if config.use_pubsub_feeds
spawn do
loop do
db.query_all("SELECT ucid FROM channels WHERE subscribed = false") do |rs|
ucid = rs.read(String)
response = subscribe_pubsub(ucid, key, config)
if response.status_code >= 400
logger.write("#{ucid} : #{response.body}\n")
end
end
sleep 1.minute
Fiber.yield
end
end
end
end
def pull_top_videos(config, db) def pull_top_videos(config, db)
loop do loop do
begin begin