From c34a24b633417084ec0f3ebfc564a55f5dfb960f Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 7 Jul 2019 14:00:42 -0500 Subject: [PATCH] Attempt to optimize query for subscription feed --- src/invidious.cr | 5 +---- src/invidious/helpers/jobs.cr | 7 ++----- src/invidious/users.cr | 13 +++++-------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index b86d46fa..3fee0a36 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1282,10 +1282,7 @@ post "/login" do |env| PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE - ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') - ORDER BY published DESC") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") if Kemal.config.ssl || config.https_only secure = true diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 38494020..b82cb1ac 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -75,7 +75,7 @@ def refresh_feeds(db, logger, config) end end - if db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "ucid = ANY" + if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))" logger.puts("Materialized view #{view_name} is out-of-date, recreating...") db.exec("DROP MATERIALIZED VIEW #{view_name}") end @@ -95,10 +95,7 @@ def refresh_feeds(db, logger, config) # While iterating through, we may have an email stored from a deleted account if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) logger.puts("CREATE #{view_name}") - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE - ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{email.gsub("'", "\\'")}') - ORDER BY published DESC") + db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex diff --git a/src/invidious/users.cr b/src/invidious/users.cr index a2143c70..c988c0c4 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -1,5 +1,8 @@ require "crypto/bcrypt/password" +# Materialized views may not be defined using bound parameters (`$1` as used elsewhere) +MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub("'", "\\'")}') ORDER BY published DESC" } + struct User module PreferencesConverter def self.from_rs(rs) @@ -166,10 +169,7 @@ def get_user(sid, headers, db, refresh = true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE - ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') - ORDER BY published DESC") + db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end @@ -188,10 +188,7 @@ def get_user(sid, headers, db, refresh = true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS \ - SELECT * FROM channel_videos WHERE - ucid IN (SELECT unnest(subscriptions) FROM users WHERE email = E'#{user.email.gsub("'", "\\'")}') - ORDER BY published DESC") + db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end