Merge 7cbf78a458 into 545a5937d8

src/invidious/jobs/statistics_refresh_job.cr

@@ -20,6 +20,11 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
     },
   }
 
+  STATISTICS_PROMETHEUS = {
+    "invidious_updated_at"                => Time.utc.to_unix,
+    "invidious_last_channel_refreshed_at" => 0_i64,
+  }
+
   private getter db : DB::Database
 
   def initialize(@db, @software_config : Hash(String, String))
@@ -52,6 +57,9 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
     users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
     users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
 
+    STATISTICS_PROMETHEUS["invidious_updated_at"] = Time.utc.to_unix
+    STATISTICS_PROMETHEUS["invidious_last_channel_refreshed_at"] = Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64
+
     STATISTICS["metadata"] = {
       "updatedAt"              => Time.utc.to_unix,
       "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,

src/invidious/metrics.cr

@@ -0,0 +1,48 @@
+# Module containing an optional Kemal handler, which can be used
+# to collect metrics about the usage of various Invidious routes.
+module Metrics
+  record MetricLabels, request_method : String, request_route : String, response_code : Int32
+
+  # Counts how many a given route was used
+  REQUEST_COUNTERS              = Hash(MetricLabels, Int64).new
+
+  # Counts how much time was used to handle requests to each route
+  REQUEST_DURATION_SECONDS_SUMS = Hash(MetricLabels, Float32).new
+
+  # The handler which will record metrics when registered in a Kemal application 
+  METRICS_COLLECTOR = RouteMetricsCollector.new(REQUEST_COUNTERS, REQUEST_DURATION_SECONDS_SUMS)
+
+  class RouteMetricsCollector < Kemal::Handler
+    def initialize(
+      @num_of_request_counters : Hash(MetricLabels, Int64),
+      @request_duration_seconds_sums : Hash(MetricLabels, Float32)
+    )
+    end
+
+    def call(context : HTTP::Server::Context)
+      request_handling_started = Time.utc
+      begin
+        call_next(context)
+      ensure
+        request_handling_finished = Time.utc
+        request_path = context.route.path
+        request_method = context.request.method
+        seconds_spent_handling = (request_handling_finished - request_handling_started).to_f
+        response_status = context.response.status_code.to_i
+
+        LOGGER.trace("Collecting metrics: handling #{request_method} #{request_path} took #{seconds_spent_handling}s and finished with status #{response_status}")
+        metric_key = MetricLabels.new request_path, request_method, response_status
+
+        unless @num_of_request_counters.has_key?(metric_key)
+          @num_of_request_counters[metric_key] = 0
+        end
+        @num_of_request_counters[metric_key] += 1
+
+        unless @request_duration_seconds_sums.has_key?(metric_key)
+          @request_duration_seconds_sums[metric_key] = 0.0
+        end
+        @request_duration_seconds_sums[metric_key] += seconds_spent_handling
+      end
+    end
+  end
+end

src/invidious/routes/api/v1/misc.cr

@@ -1,3 +1,5 @@
+require "../../../metrics.cr"
+
 module Invidious::Routes::API::V1::Misc
   # Stats API endpoint for Invidious
   def self.stats(env)
@@ -10,6 +12,34 @@ module Invidious::Routes::API::V1::Misc
     end
   end
 
+  def self.metrics(env)
+    env.response.content_type = "text/plain"
+
+    return String.build do |str|
+      Metrics::REQUEST_COUNTERS.each do |metric_labels, value|
+        str << "http_requests_total{"
+        str << "method=\"" << metric_labels.request_method << "\" "
+        str << "route=\"" << metric_labels.request_route << "\" "
+        str << "response_code=\"" << metric_labels.response_code << "\""
+        str << "} "
+        str << value << "\n"
+      end
+
+      Metrics::REQUEST_DURATION_SECONDS_SUMS.each do |metric_labels, value|
+        str << "http_request_duration_seconds_sum{"
+        str << "method=\"" << metric_labels.request_method << "\" "
+        str << "route=\"" << metric_labels.request_route << "\" "
+        str << "response_code=\"" << metric_labels.response_code << "\""
+        str << "} "
+        str << value << "\n"
+      end
+
+      Invidious::Jobs::StatisticsRefreshJob::STATISTICS_PROMETHEUS.each.each do |key, value|
+        str << key << " " << value << "\n"
+      end
+    end
+  end
+
   # APIv1 currently uses the same logic for both
   # user playlists and Invidious playlists. This means that we can't
   # reasonably split them yet. This should be addressed in APIv2

src/invidious/routing.cr

@@ -287,6 +287,10 @@ module Invidious::Routing
 
       # Misc
       get "/api/v1/stats", {{namespace}}::Misc, :stats
+      if CONFIG.statistics_enabled
+        add_handler Metrics::METRICS_COLLECTOR
+        get "/api/v1/metrics", {{namespace}}::Misc, :metrics
+      end
       get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
       get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
       get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes