From ce8b83ab1897c3fdaf8987262bc56e974d738adb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 21 Oct 2022 00:22:31 +0200 Subject: [PATCH 1/4] Shards: Add required dependencies and update lock file --- shard.lock | 8 ++++++++ shard.yml | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index 235e4c25..ead94292 100644 --- a/shard.lock +++ b/shard.lock @@ -32,6 +32,10 @@ shards: git: https://github.com/will/crystal-pg.git version: 0.24.0 + pool: + git: https://github.com/ysbaddaden/pool.git + version: 0.2.4 + protodec: git: https://github.com/iv-org/protodec.git version: 0.1.5 @@ -40,6 +44,10 @@ shards: git: https://github.com/luislavena/radix.git version: 0.4.1 + redis: + git: https://github.com/stefanwille/crystal-redis.git + version: 2.8.3 + spectator: git: https://github.com/icy-arctic-fox/spectator.git version: 0.10.4 diff --git a/shard.yml b/shard.yml index 7ee0bb2a..6533170b 100644 --- a/shard.yml +++ b/shard.yml @@ -10,27 +10,38 @@ targets: main: src/invidious.cr dependencies: + # Database pg: github: will/crystal-pg version: ~> 0.24.0 sqlite3: github: crystal-lang/crystal-sqlite3 version: ~> 0.18.0 + + # Web server kemal: github: kemalcr/kemal version: ~> 1.1.2 kilt: github: jeromegn/kilt version: ~> 0.6.1 + athena-negotiation: + github: athena-framework/negotiation + version: ~> 0.1.1 + + # Youtube backend protodec: github: iv-org/protodec version: ~> 0.1.5 lsquic: github: iv-org/lsquic.cr version: ~> 2.18.1-2 - athena-negotiation: - github: athena-framework/negotiation - version: ~> 0.1.1 + + # Caching + redis: + github: stefanwille/crystal-redis + version: ~> 2.8.3 + development_dependencies: spectator: From 8bbc46fd981229962af843c0f04cdefe46c44c73 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 3 Apr 2023 00:03:06 +0200 Subject: [PATCH 2/4] Config: Add scheme support to DBConfig --- src/invidious/config.cr | 13 +++---------- src/invidious/config/db.cr | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 src/invidious/config/db.cr diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 9fc58409..66ba9df8 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -1,12 +1,5 @@ -struct DBConfig - include YAML::Serializable - - property user : String - property password : String - property host : String - property port : Int32 - property dbname : String -end +require "yaml" +require "./config/*" struct ConfigPreferences include YAML::Serializable @@ -69,7 +62,7 @@ class Config # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info # Database configuration with separate parameters (username, hostname, etc) - property db : DBConfig? = nil + property db : IV::Config::DBConfig? = nil # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] diff --git a/src/invidious/config/db.cr b/src/invidious/config/db.cr new file mode 100644 index 00000000..7ee3b9c6 --- /dev/null +++ b/src/invidious/config/db.cr @@ -0,0 +1,23 @@ +module Invidious::Config + struct DBConfig + include YAML::Serializable + + property scheme : String + property user : String + property password : String + property host : String + property port : Int32 + property dbname : String + + def to_uri + return URI.new( + scheme: @scheme, + user: @user, + password: @password, + host: @host, + port: @port, + path: @dbname, + ) + end + end +end From 53f50611c2be7948abbc93f4d5f87d588032a76b Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 3 Apr 2023 00:12:36 +0200 Subject: [PATCH 3/4] Config: clean up the various converters --- src/invidious/config.cr | 9 +- src/invidious/config/converters.cr | 74 ++++++++++++++ src/invidious/user/preferences.cr | 153 ----------------------------- 3 files changed, 79 insertions(+), 157 deletions(-) create mode 100644 src/invidious/config/converters.cr diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 66ba9df8..2ade568c 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -53,7 +53,7 @@ class Config # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Time interval between two executions of the job that crawls channel videos (subscriptions update). - @[YAML::Field(converter: Preferences::TimeSpanConverter)] + @[YAML::Field(converter: IV::Config::TimeSpanConverter)] property channel_refresh_interval : Time::Span = 30.minutes # Number of threads to use for updating feeds property feed_threads : Int32 = 1 @@ -65,7 +65,7 @@ class Config property db : IV::Config::DBConfig? = nil # Database configuration using 12-Factor "Database URL" syntax - @[YAML::Field(converter: Preferences::URIConverter)] + @[YAML::Field(converter: IV::Config::URIConverter)] property database_url : URI = URI.parse("") # Use polling to keep decryption function up to date property decrypt_polling : Bool = false @@ -111,8 +111,9 @@ class Config property modified_source_code_url : String? = nil # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - @[YAML::Field(converter: Preferences::FamilyConverter)] + @[YAML::Field(converter: IV::Config::FamilyConverter)] property force_resolve : Socket::Family = Socket::Family::UNSPEC + # Port to listen for connections (overridden by command line argument) property port : Int32 = 3000 # Host to bind (overridden by command line argument) @@ -123,7 +124,7 @@ class Config property use_quic : Bool = false # Saved cookies in "name1=value1; name2=value2..." format - @[YAML::Field(converter: Preferences::StringToCookies)] + @[YAML::Field(converter: IV::Config::CookiesConverter)] property cookies : HTTP::Cookies = HTTP::Cookies.new # Key for Anti-Captcha property captcha_key : String? = nil diff --git a/src/invidious/config/converters.cr b/src/invidious/config/converters.cr new file mode 100644 index 00000000..62a04fc2 --- /dev/null +++ b/src/invidious/config/converters.cr @@ -0,0 +1,74 @@ +module Invidious::Config + module CookiesConverter + def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) + (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + cookies = HTTP::Cookies.new + node.value.split(";").each do |cookie| + next if cookie.strip.empty? + name, value = cookie.split("=", 2) + cookies << HTTP::Cookie.new(name.strip, value.strip) + end + + return cookies + end + end + + module FamilyConverter + def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) + case value + when Socket::Family::UNSPEC then yaml.scalar nil + when Socket::Family::INET then yaml.scalar "ipv4" + when Socket::Family::INET6 then yaml.scalar "ipv6" + when Socket::Family::UNIX then raise "Invalid socket family #{value}" + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family + if node.is_a?(YAML::Nodes::Scalar) + case node.value.downcase + when "ipv4" then Socket::Family::INET + when "ipv6" then Socket::Family::INET6 + else + Socket::Family::UNSPEC + end + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module URIConverter + def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) + yaml.scalar value.normalize! + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI + if node.is_a?(YAML::Nodes::Scalar) + URI.parse node.value + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module TimeSpanConverter + def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) + return yaml.scalar value.total_minutes.to_i32 + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span + if node.is_a?(YAML::Nodes::Scalar) + return decode_interval(node.value) + else + node.raise "Expected scalar, not #{node.class}" + end + end + end +end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index b3059403..c8c71f8a 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -1,6 +1,5 @@ struct Preferences include JSON::Serializable - include YAML::Serializable property annotations : Bool = CONFIG.default_user_preferences.annotations property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed @@ -8,17 +7,14 @@ struct Preferences property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect @[JSON::Field(converter: Preferences::StringToArray)] - @[YAML::Field(converter: Preferences::StringToArray)] property captions : Array(String) = CONFIG.default_user_preferences.captions @[JSON::Field(converter: Preferences::StringToArray)] - @[YAML::Field(converter: Preferences::StringToArray)] property comments : Array(String) = CONFIG.default_user_preferences.comments property continue : Bool = CONFIG.default_user_preferences.continue property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay @[JSON::Field(converter: Preferences::BoolToString)] - @[YAML::Field(converter: Preferences::BoolToString)] property dark_mode : String = CONFIG.default_user_preferences.dark_mode property latest_only : Bool = CONFIG.default_user_preferences.latest_only property listen : Bool = CONFIG.default_user_preferences.listen @@ -78,27 +74,6 @@ struct Preferences end end end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - case node.value - when "true" - "dark" - when "false" - "light" - when "" - CONFIG.default_user_preferences.dark_mode - else - node.value - end - end end module ClampInt @@ -109,58 +84,6 @@ struct Preferences def self.from_json(value : JSON::PullParser) : Int32 value.read_int.clamp(0, MAX_ITEMS_PER_PAGE).to_i32 end - - def self.to_yaml(value : Int32, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Int32 - node.value.clamp(0, MAX_ITEMS_PER_PAGE) - end - end - - module FamilyConverter - def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) - case value - when Socket::Family::UNSPEC - yaml.scalar nil - when Socket::Family::INET - yaml.scalar "ipv4" - when Socket::Family::INET6 - yaml.scalar "ipv6" - when Socket::Family::UNIX - raise "Invalid socket family #{value}" - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family - if node.is_a?(YAML::Nodes::Scalar) - case node.value.downcase - when "ipv4" - Socket::Family::INET - when "ipv6" - Socket::Family::INET6 - else - Socket::Family::UNSPEC - end - else - node.raise "Expected scalar, not #{node.class}" - end - end - end - - module URIConverter - def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) - yaml.scalar value.normalize! - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI - if node.is_a?(YAML::Nodes::Scalar) - URI.parse node.value - else - node.raise "Expected scalar, not #{node.class}" - end - end end module ProcessString @@ -171,14 +94,6 @@ struct Preferences def self.from_json(value : JSON::PullParser) : String HTML.escape(value.read_string[0, 100]) end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - HTML.escape(node.value[0, 100]) - end end module StringToArray @@ -202,73 +117,5 @@ struct Preferences result end - - def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) - yaml.sequence do - value.each do |element| - yaml.scalar element - end - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) - begin - unless node.is_a?(YAML::Nodes::Sequence) - node.raise "Expected sequence, not #{node.class}" - end - - result = [] of String - node.nodes.each do |item| - unless item.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{item.class}" - end - - result << HTML.escape(item.value[0, 100]) - end - rescue ex - if node.is_a?(YAML::Nodes::Scalar) - result = [HTML.escape(node.value[0, 100]), ""] - else - result = ["", ""] - end - end - - result - end - end - - module StringToCookies - def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) - (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - cookies = HTTP::Cookies.new - node.value.split(";").each do |cookie| - next if cookie.strip.empty? - name, value = cookie.split("=", 2) - cookies << HTTP::Cookie.new(name.strip, value.strip) - end - - cookies - end - end - - module TimeSpanConverter - def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) - return yaml.scalar value.total_minutes.to_i32 - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span - if node.is_a?(YAML::Nodes::Scalar) - return decode_interval(node.value) - else - node.raise "Expected scalar, not #{node.class}" - end - end end end From 890aebdc1dce885619f4425f717004569b6a446a Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 23 Oct 2022 14:15:16 +0200 Subject: [PATCH 4/4] Create the base of the cache subsystem --- src/invidious/cache.cr | 27 +++++++++ src/invidious/cache/cacheable_item.cr | 9 +++ src/invidious/cache/item_store.cr | 22 +++++++ src/invidious/cache/null_item_store.cr | 24 ++++++++ src/invidious/cache/postgres_item_store.cr | 70 ++++++++++++++++++++++ src/invidious/cache/redis_item_store.cr | 36 +++++++++++ src/invidious/config.cr | 12 ++-- src/invidious/config/cache.cr | 17 ++++++ 8 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 src/invidious/cache.cr create mode 100644 src/invidious/cache/cacheable_item.cr create mode 100644 src/invidious/cache/item_store.cr create mode 100644 src/invidious/cache/null_item_store.cr create mode 100644 src/invidious/cache/postgres_item_store.cr create mode 100644 src/invidious/cache/redis_item_store.cr create mode 100644 src/invidious/config/cache.cr diff --git a/src/invidious/cache.cr b/src/invidious/cache.cr new file mode 100644 index 00000000..16acfa19 --- /dev/null +++ b/src/invidious/cache.cr @@ -0,0 +1,27 @@ +require "./cache/*" + +module Invidious::Cache + extend self + + INSTANCE = self.init(CONFIG.cache) + + def init(cfg : Config::CacheConfig) : ItemStore + return NullItemStore.new if !cfg.enabled + + # Environment variable takes precedence over local config + url = ENV.get?("INVIDIOUS__CACHE__URL").try { |u| URI.parse(u) } + url ||= CONFIG.cache.url + + case type + when .postgres? + # Use the database URL as a compatibility fallback + url ||= CONFIG.database_url + return PostgresItemStore.new(url) + when .redis? + raise InvalidConfigException.new "Redis cache requires an URL." if url.nil? + return RedisItemStore.new(url) + else + return NullItemStore.new + end + end +end diff --git a/src/invidious/cache/cacheable_item.cr b/src/invidious/cache/cacheable_item.cr new file mode 100644 index 00000000..c1295a4a --- /dev/null +++ b/src/invidious/cache/cacheable_item.cr @@ -0,0 +1,9 @@ +require "json" + +module Invidious::Cache + # Including this module allows the includer object to be cached. + # The object will automatically inherit from JSON::Serializable. + module CacheableItem + include JSON::Serializable + end +end diff --git a/src/invidious/cache/item_store.cr b/src/invidious/cache/item_store.cr new file mode 100644 index 00000000..e4ec1201 --- /dev/null +++ b/src/invidious/cache/item_store.cr @@ -0,0 +1,22 @@ +require "./cacheable_item" + +module Invidious::Cache + # Abstract class from which any cached element should inherit + # Note: class is used here, instead of a module, in order to benefit + # from various compiler checks (e.g methods must be implemented) + abstract class ItemStore + # Retrieves an item from the store + # Returns nil if item wasn't found or is expired + abstract def fetch(key : String, *, as : T.class) + + # Stores a given item into cache + abstract def store(key : String, value : CacheableItem, expires : Time::Span) + + # Prematurely deletes item(s) from the cache + abstract def delete(key : String) + abstract def delete(keys : Array(String)) + + # Removes all the items stored in the cache + abstract def clear + end +end diff --git a/src/invidious/cache/null_item_store.cr b/src/invidious/cache/null_item_store.cr new file mode 100644 index 00000000..c26c0804 --- /dev/null +++ b/src/invidious/cache/null_item_store.cr @@ -0,0 +1,24 @@ +require "./item_store" + +module Invidious::Cache + class NullItemStore < ItemStore + def initialize + end + + def fetch(key : String, *, as : T.class) : T? forall T + return nil + end + + def store(key : String, value : CacheableItem, expires : Time::Span) + end + + def delete(key : String) + end + + def delete(keys : Array(String)) + end + + def clear + end + end +end diff --git a/src/invidious/cache/postgres_item_store.cr b/src/invidious/cache/postgres_item_store.cr new file mode 100644 index 00000000..cfbe52e2 --- /dev/null +++ b/src/invidious/cache/postgres_item_store.cr @@ -0,0 +1,70 @@ +require "./item_store" +require "json" +require "pg" + +module Invidious::Cache + class PostgresItemStore < ItemStore + @db : DB::Database + @node_name : String + + def initialize(url : URI, @node_name = "") + @db = DB.open url + end + + def fetch(key : String, *, as : T.class) : T? forall T + request = <<-SQL + SELECT info,updated + FROM videos + WHERE id = $1 + SQL + + value, expires = @db.query_one?(request, key, as: {String?, Time?}) + + if expires < Time.utc + self.delete(key) + return nil + else + return T.from_json(JSON::PullParser.new(value)) + end + end + + def store(key : String, value : CacheableItem, expires : Time::Span) + request = <<-SQL + INSERT INTO videos + VALUES ($1, $2, $3) + ON CONFLICT (id) DO + UPDATE + SET info = $2, updated = $3 + SQL + + @db.exec(request, key, value.to_json, Time.utc + expires) + end + + def delete(key : String) + request = <<-SQL + DELETE FROM videos * + WHERE id = $1 + SQL + + @db.exec(request, key) + end + + def delete(keys : Array(String)) + request = <<-SQL + DELETE FROM videos * + WHERE id = ANY($1::TEXT[]) + SQL + + @db.exec(request, keys) + end + + def clear + request = <<-SQL + DELETE FROM videos * + WHERE updated < now() + SQL + + @db.exec(request) + end + end +end diff --git a/src/invidious/cache/redis_item_store.cr b/src/invidious/cache/redis_item_store.cr new file mode 100644 index 00000000..ccf847a6 --- /dev/null +++ b/src/invidious/cache/redis_item_store.cr @@ -0,0 +1,36 @@ +require "./item_store" +require "json" +require "redis" + +module Invidious::Cache + class RedisItemStore < ItemStore + @redis : Redis::PooledClient + @node_name : String + + def initialize(url : URI, @node_name = "") + @redis = Redis::PooledClient.new url + end + + def fetch(key : String, *, as : T.class) : (T | Nil) forall T + value = @redis.get(key) + return nil if value.nil? + return T.from_json(JSON::PullParser.new(value)) + end + + def store(key : String, value : CacheableItem, expires : Time::Span) + @redis.set(key, value, ex: expires.to_i) + end + + def delete(key : String) + @redis.del(key) + end + + def delete(keys : Array(String)) + @redis.del(keys) + end + + def clear + @redis.flushdb + end + end +end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 2ade568c..f180cf24 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -74,6 +74,8 @@ class Config # Jobs config structure. See jobs.cr and jobs/base_job.cr property jobs = Invidious::Jobs::JobsConfig.new + # Cache configuration. See cache/cache.cr + property cache = Invidious::Config::CacheConfig.new # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? @@ -201,14 +203,8 @@ class Config # Build database_url from db.* if it's not set directly if config.database_url.to_s.empty? if db = config.db - config.database_url = URI.new( - scheme: "postgres", - user: db.user, - password: db.password, - host: db.host, - port: db.port, - path: db.dbname, - ) + db.scheme = "postgres" + config.database_url = db.to_uri else puts "Config : Either database_url or db.* is required" exit(1) diff --git a/src/invidious/config/cache.cr b/src/invidious/config/cache.cr new file mode 100644 index 00000000..d629c115 --- /dev/null +++ b/src/invidious/config/cache.cr @@ -0,0 +1,17 @@ +require "../cache/store_type" + +module Invidious::Config + struct CacheConfig + include YAML::Serializable + + getter enabled : Bool = true + getter type : Cache::StoreType = :postgres + + @[YAML::Field(converter: IV::Config::URIConverter)] + @url : URI? = URI.parse("") + + # Required because of YAML serialization + def initialize + end + end +end