Merge 890aebdc1d into 545a5937d8

shard.lock

@@ -32,6 +32,10 @@ shards:
     git: https://github.com/will/crystal-pg.git
     version: 0.24.0
 
+  pool:
+    git: https://github.com/ysbaddaden/pool.git
+    version: 0.2.4
+
   protodec:
     git: https://github.com/iv-org/protodec.git
     version: 0.1.5
@@ -40,6 +44,10 @@ shards:
     git: https://github.com/luislavena/radix.git
     version: 0.4.1
 
+  redis:
+    git: https://github.com/stefanwille/crystal-redis.git
+    version: 2.8.3
+
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
     version: 0.10.4

shard.yml

@@ -10,27 +10,38 @@ targets:
     main: src/invidious.cr
 
 dependencies:
+  # Database
   pg:
     github: will/crystal-pg
     version: ~> 0.24.0
   sqlite3:
     github: crystal-lang/crystal-sqlite3
     version: ~> 0.18.0
+
+  # Web server
   kemal:
     github: kemalcr/kemal
     version: ~> 1.1.2
   kilt:
     github: jeromegn/kilt
     version: ~> 0.6.1
+  athena-negotiation:
+    github: athena-framework/negotiation
+    version: ~> 0.1.1
+
+  # Youtube backend
   protodec:
     github: iv-org/protodec
     version: ~> 0.1.5
   lsquic:
     github: iv-org/lsquic.cr
     version: ~> 2.18.1-2
-  athena-negotiation:
+
-    github: athena-framework/negotiation
+  # Caching
-    version: ~> 0.1.1
+  redis:
+    github: stefanwille/crystal-redis
+    version: ~> 2.8.3
+
 
 development_dependencies:
   spectator:

src/invidious/cache.cr

@@ -0,0 +1,27 @@
+require "./cache/*"
+
+module Invidious::Cache
+  extend self
+
+  INSTANCE = self.init(CONFIG.cache)
+
+  def init(cfg : Config::CacheConfig) : ItemStore
+    return NullItemStore.new if !cfg.enabled
+
+    # Environment variable takes precedence over local config
+    url = ENV.get?("INVIDIOUS__CACHE__URL").try { |u| URI.parse(u) }
+    url ||= CONFIG.cache.url
+
+    case type
+    when .postgres?
+      # Use the database URL as a compatibility fallback
+      url ||= CONFIG.database_url
+      return PostgresItemStore.new(url)
+    when .redis?
+      raise InvalidConfigException.new "Redis cache requires an URL." if url.nil?
+      return RedisItemStore.new(url)
+    else
+      return NullItemStore.new
+    end
+  end
+end

src/invidious/cache/cacheable_item.cr

@@ -0,0 +1,9 @@
+require "json"
+
+module Invidious::Cache
+  # Including this module allows the includer object to be cached.
+  # The object will automatically inherit from JSON::Serializable.
+  module CacheableItem
+    include JSON::Serializable
+  end
+end

src/invidious/cache/item_store.cr

@@ -0,0 +1,22 @@
+require "./cacheable_item"
+
+module Invidious::Cache
+  # Abstract class from which any cached element should inherit
+  # Note: class is used here, instead of a module, in order to benefit
+  # from various compiler checks (e.g methods must be implemented)
+  abstract class ItemStore
+    # Retrieves an item from the store
+    # Returns nil if item wasn't found or is expired
+    abstract def fetch(key : String, *, as : T.class)
+
+    # Stores a given item into cache
+    abstract def store(key : String, value : CacheableItem, expires : Time::Span)
+
+    # Prematurely deletes item(s) from the cache
+    abstract def delete(key : String)
+    abstract def delete(keys : Array(String))
+
+    # Removes all the items stored in the cache
+    abstract def clear
+  end
+end

src/invidious/cache/null_item_store.cr

@@ -0,0 +1,24 @@
+require "./item_store"
+
+module Invidious::Cache
+  class NullItemStore < ItemStore
+    def initialize
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      return nil
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+    end
+
+    def delete(key : String)
+    end
+
+    def delete(keys : Array(String))
+    end
+
+    def clear
+    end
+  end
+end

src/invidious/cache/postgres_item_store.cr

@@ -0,0 +1,70 @@
+require "./item_store"
+require "json"
+require "pg"
+
+module Invidious::Cache
+  class PostgresItemStore < ItemStore
+    @db : DB::Database
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @db = DB.open url
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      request = <<-SQL
+        SELECT info,updated
+        FROM videos
+        WHERE id = $1
+      SQL
+
+      value, expires = @db.query_one?(request, key, as: {String?, Time?})
+
+      if expires < Time.utc
+        self.delete(key)
+        return nil
+      else
+        return T.from_json(JSON::PullParser.new(value))
+      end
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      request = <<-SQL
+        INSERT INTO videos
+        VALUES ($1, $2, $3)
+        ON CONFLICT (id) DO
+          UPDATE
+          SET info = $2, updated = $3
+      SQL
+
+      @db.exec(request, key, value.to_json, Time.utc + expires)
+    end
+
+    def delete(key : String)
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = $1
+      SQL
+
+      @db.exec(request, key)
+    end
+
+    def delete(keys : Array(String))
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = ANY($1::TEXT[])
+      SQL
+
+      @db.exec(request, keys)
+    end
+
+    def clear
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE updated < now()
+      SQL
+
+      @db.exec(request)
+    end
+  end
+end

src/invidious/cache/redis_item_store.cr

@@ -0,0 +1,36 @@
+require "./item_store"
+require "json"
+require "redis"
+
+module Invidious::Cache
+  class RedisItemStore < ItemStore
+    @redis : Redis::PooledClient
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @redis = Redis::PooledClient.new url
+    end
+
+    def fetch(key : String, *, as : T.class) : (T | Nil) forall T
+      value = @redis.get(key)
+      return nil if value.nil?
+      return T.from_json(JSON::PullParser.new(value))
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      @redis.set(key, value, ex: expires.to_i)
+    end
+
+    def delete(key : String)
+      @redis.del(key)
+    end
+
+    def delete(keys : Array(String))
+      @redis.del(keys)
+    end
+
+    def clear
+      @redis.flushdb
+    end
+  end
+end

src/invidious/config.cr

@@ -1,12 +1,5 @@
-struct DBConfig
+require "yaml"
-  include YAML::Serializable
+require "./config/*"
-
-  property user : String
-  property password : String
-  property host : String
-  property port : Int32
-  property dbname : String
-end
 
 struct ConfigPreferences
   include YAML::Serializable
@@ -60,7 +53,7 @@ class Config
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
-  @[YAML::Field(converter: Preferences::TimeSpanConverter)]
+  @[YAML::Field(converter: IV::Config::TimeSpanConverter)]
   property channel_refresh_interval : Time::Span = 30.minutes
   # Number of threads to use for updating feeds
   property feed_threads : Int32 = 1
@@ -69,10 +62,10 @@ class Config
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
   property log_level : LogLevel = LogLevel::Info
   # Database configuration with separate parameters (username, hostname, etc)
-  property db : DBConfig? = nil
+  property db : IV::Config::DBConfig? = nil
 
   # Database configuration using 12-Factor "Database URL" syntax
-  @[YAML::Field(converter: Preferences::URIConverter)]
+  @[YAML::Field(converter: IV::Config::URIConverter)]
   property database_url : URI = URI.parse("")
   # Use polling to keep decryption function up to date
   property decrypt_polling : Bool = false
@@ -81,6 +74,8 @@ class Config
 
   # Jobs config structure. See jobs.cr and jobs/base_job.cr
   property jobs = Invidious::Jobs::JobsConfig.new
+  # Cache configuration. See cache/cache.cr
+  property cache = Invidious::Config::CacheConfig.new
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
@@ -118,8 +113,9 @@ class Config
   property modified_source_code_url : String? = nil
 
   # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
-  @[YAML::Field(converter: Preferences::FamilyConverter)]
+  @[YAML::Field(converter: IV::Config::FamilyConverter)]
   property force_resolve : Socket::Family = Socket::Family::UNSPEC
+
   # Port to listen for connections (overridden by command line argument)
   property port : Int32 = 3000
   # Host to bind (overridden by command line argument)
@@ -130,7 +126,7 @@ class Config
   property use_quic : Bool = false
 
   # Saved cookies in "name1=value1; name2=value2..." format
-  @[YAML::Field(converter: Preferences::StringToCookies)]
+  @[YAML::Field(converter: IV::Config::CookiesConverter)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
   # Key for Anti-Captcha
   property captcha_key : String? = nil
@@ -207,14 +203,8 @@ class Config
     # Build database_url from db.* if it's not set directly
     if config.database_url.to_s.empty?
       if db = config.db
-        config.database_url = URI.new(
+        db.scheme = "postgres"
-          scheme: "postgres",
+        config.database_url = db.to_uri
-          user: db.user,
-          password: db.password,
-          host: db.host,
-          port: db.port,
-          path: db.dbname,
-        )
       else
         puts "Config : Either database_url or db.* is required"
         exit(1)

src/invidious/config/cache.cr

@@ -0,0 +1,17 @@
+require "../cache/store_type"
+
+module Invidious::Config
+  struct CacheConfig
+    include YAML::Serializable
+
+    getter enabled : Bool = true
+    getter type : Cache::StoreType = :postgres
+
+    @[YAML::Field(converter: IV::Config::URIConverter)]
+    @url : URI? = URI.parse("")
+
+    # Required because of YAML serialization
+    def initialize
+    end
+  end
+end

src/invidious/config/converters.cr

@@ -0,0 +1,74 @@
+module Invidious::Config
+  module CookiesConverter
+    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+      unless node.is_a?(YAML::Nodes::Scalar)
+        node.raise "Expected scalar, not #{node.class}"
+      end
+
+      cookies = HTTP::Cookies.new
+      node.value.split(";").each do |cookie|
+        next if cookie.strip.empty?
+        name, value = cookie.split("=", 2)
+        cookies << HTTP::Cookie.new(name.strip, value.strip)
+      end
+
+      return cookies
+    end
+  end
+
+  module FamilyConverter
+    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+      case value
+      when Socket::Family::UNSPEC then yaml.scalar nil
+      when Socket::Family::INET   then yaml.scalar "ipv4"
+      when Socket::Family::INET6  then yaml.scalar "ipv6"
+      when Socket::Family::UNIX   then raise "Invalid socket family #{value}"
+      end
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+      if node.is_a?(YAML::Nodes::Scalar)
+        case node.value.downcase
+        when "ipv4" then Socket::Family::INET
+        when "ipv6" then Socket::Family::INET6
+        else
+          Socket::Family::UNSPEC
+        end
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+
+  module URIConverter
+    def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder)
+      yaml.scalar value.normalize!
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI
+      if node.is_a?(YAML::Nodes::Scalar)
+        URI.parse node.value
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+
+  module TimeSpanConverter
+    def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder)
+      return yaml.scalar value.total_minutes.to_i32
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span
+      if node.is_a?(YAML::Nodes::Scalar)
+        return decode_interval(node.value)
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+end

src/invidious/config/db.cr

@@ -0,0 +1,23 @@
+module Invidious::Config
+  struct DBConfig
+    include YAML::Serializable
+
+    property scheme : String
+    property user : String
+    property password : String
+    property host : String
+    property port : Int32
+    property dbname : String
+
+    def to_uri
+      return URI.new(
+        scheme: @scheme,
+        user: @user,
+        password: @password,
+        host: @host,
+        port: @port,
+        path: @dbname,
+      )
+    end
+  end
+end

src/invidious/user/preferences.cr

@@ -1,6 +1,5 @@
 struct Preferences
   include JSON::Serializable
-  include YAML::Serializable
 
   property annotations : Bool = CONFIG.default_user_preferences.annotations
   property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
@@ -8,17 +7,14 @@ struct Preferences
   property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
 
   @[JSON::Field(converter: Preferences::StringToArray)]
-  @[YAML::Field(converter: Preferences::StringToArray)]
   property captions : Array(String) = CONFIG.default_user_preferences.captions
 
   @[JSON::Field(converter: Preferences::StringToArray)]
-  @[YAML::Field(converter: Preferences::StringToArray)]
   property comments : Array(String) = CONFIG.default_user_preferences.comments
   property continue : Bool = CONFIG.default_user_preferences.continue
   property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
 
   @[JSON::Field(converter: Preferences::BoolToString)]
-  @[YAML::Field(converter: Preferences::BoolToString)]
   property dark_mode : String = CONFIG.default_user_preferences.dark_mode
   property latest_only : Bool = CONFIG.default_user_preferences.latest_only
   property listen : Bool = CONFIG.default_user_preferences.listen
@@ -78,27 +74,6 @@ struct Preferences
         end
       end
     end
-
-    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      case node.value
-      when "true"
-        "dark"
-      when "false"
-        "light"
-      when ""
-        CONFIG.default_user_preferences.dark_mode
-      else
-        node.value
-      end
-    end
   end
 
   module ClampInt
@@ -109,58 +84,6 @@ struct Preferences
     def self.from_json(value : JSON::PullParser) : Int32
       value.read_int.clamp(0, MAX_ITEMS_PER_PAGE).to_i32
     end
-
-    def self.to_yaml(value : Int32, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Int32
-      node.value.clamp(0, MAX_ITEMS_PER_PAGE)
-    end
-  end
-
-  module FamilyConverter
-    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
-      case value
-      when Socket::Family::UNSPEC
-        yaml.scalar nil
-      when Socket::Family::INET
-        yaml.scalar "ipv4"
-      when Socket::Family::INET6
-        yaml.scalar "ipv6"
-      when Socket::Family::UNIX
-        raise "Invalid socket family #{value}"
-      end
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
-      if node.is_a?(YAML::Nodes::Scalar)
-        case node.value.downcase
-        when "ipv4"
-          Socket::Family::INET
-        when "ipv6"
-          Socket::Family::INET6
-        else
-          Socket::Family::UNSPEC
-        end
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
-  end
-
-  module URIConverter
-    def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder)
-      yaml.scalar value.normalize!
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI
-      if node.is_a?(YAML::Nodes::Scalar)
-        URI.parse node.value
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
   end
 
   module ProcessString
@@ -171,14 +94,6 @@ struct Preferences
     def self.from_json(value : JSON::PullParser) : String
       HTML.escape(value.read_string[0, 100])
     end
-
-    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      HTML.escape(node.value[0, 100])
-    end
   end
 
   module StringToArray
@@ -202,73 +117,5 @@ struct Preferences
 
       result
     end
-
-    def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
-      yaml.sequence do
-        value.each do |element|
-          yaml.scalar element
-        end
-      end
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
-      begin
-        unless node.is_a?(YAML::Nodes::Sequence)
-          node.raise "Expected sequence, not #{node.class}"
-        end
-
-        result = [] of String
-        node.nodes.each do |item|
-          unless item.is_a?(YAML::Nodes::Scalar)
-            node.raise "Expected scalar, not #{item.class}"
-          end
-
-          result << HTML.escape(item.value[0, 100])
-        end
-      rescue ex
-        if node.is_a?(YAML::Nodes::Scalar)
-          result = [HTML.escape(node.value[0, 100]), ""]
-        else
-          result = ["", ""]
-        end
-      end
-
-      result
-    end
-  end
-
-  module StringToCookies
-    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
-      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      cookies = HTTP::Cookies.new
-      node.value.split(";").each do |cookie|
-        next if cookie.strip.empty?
-        name, value = cookie.split("=", 2)
-        cookies << HTTP::Cookie.new(name.strip, value.strip)
-      end
-
-      cookies
-    end
-  end
-
-  module TimeSpanConverter
-    def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder)
-      return yaml.scalar value.total_minutes.to_i32
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span
-      if node.is_a?(YAML::Nodes::Scalar)
-        return decode_interval(node.value)
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
   end
 end