Add backtraces to errors (#1498)

Error handling has been reworked to always go through the new `error_template`,
`error_json` and `error_atom` macros.
They all accept a status code followed by a string message or an exception
object. `error_json` accepts a hash with additional fields as third argument.

If the second argument is an exception a backtrace will be printed, if it is a
string only the string is printed. Since up till now only the exception message
was printed a new `InfoException` class was added for situations where no
backtrace is intended but a string cannot be used.

`error_template` with a string message automatically localizes the message.
Missing error translations have been collected in https://github.com/iv-org/invidious/issues/1497
`error_json` with a string message does not localize the message. This is the
same as previous behavior. If translations are desired for `error_json` they
can be added easily but those error messages have not been collected yet.

Uncaught exceptions previously only printed a generic message ("Looks like
you've found a bug in Invidious. [...]"). They still print that message
but now also include a backtrace.

src/invidious/channels.cr

@@ -208,7 +208,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 
   author = rss.xpath_node(%q(//feed/title))
   if !author
-    raise translate(locale, "Deleted or invalid channel")
+    raise InfoException.new("Deleted or invalid channel")
   end
   author = author.content
 
@@ -226,13 +226,14 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
   videos = [] of SearchVideo
   begin
     initial_data = JSON.parse(response.body).as_a.find &.["response"]?
-    raise "Could not extract JSON" if !initial_data
+    raise InfoException.new("Could not extract channel JSON") if !initial_data
     videos = extract_videos(initial_data.as_h, author, ucid)
   rescue ex
     if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
        response.body.includes?("https://www.google.com/sorry/index")
-      raise "Could not extract channel info. Instance is likely blocked."
+      raise InfoException.new("Could not extract channel info. Instance is likely blocked.")
     end
+    raise ex
   end
 
   rss.xpath_nodes("//feed/entry").each do |entry|
@@ -287,7 +288,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
     loop do
       response = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
       initial_data = JSON.parse(response.body).as_a.find &.["response"]?
-      raise "Could not extract JSON" if !initial_data
+      raise InfoException.new("Could not extract channel JSON") if !initial_data
       videos = extract_videos(initial_data.as_h, author, ucid)
 
       count = videos.size
@@ -507,8 +508,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
   end
 
   if response.status_code != 200
-    error_message = translate(locale, "This channel does not exist.")
-    raise error_message
+    raise InfoException.new("This channel does not exist.")
   end
 
   ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"]
@@ -518,7 +518,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
     body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]?
 
     if !body
-      raise "Could not extract community tab."
+      raise InfoException.new("Could not extract community tab.")
     end
 
     body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]
@@ -540,7 +540,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
            body["response"]["continuationContents"]["backstageCommentsContinuation"]?
 
     if !body
-      raise "Could not extract continuation."
+      raise InfoException.new("Could not extract continuation.")
     end
   end
 
@@ -551,7 +551,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
     error_message = (message["text"]["simpleText"]? ||
                      message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
       .try &.as_s || ""
-    raise error_message
+    raise InfoException.new(error_message)
   end
 
   response = JSON.build do |json|
@@ -786,21 +786,19 @@ def get_about_info(ucid, locale)
   end
 
   if result.status_code != 200
-    error_message = translate(locale, "This channel does not exist.")
-    raise error_message
+    raise InfoException.new("This channel does not exist.")
   end
 
   about = XML.parse_html(result.body)
   if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")]))
-    error_message = translate(locale, "This channel does not exist.")
-    raise error_message
+    raise InfoException.new("This channel does not exist.")
   end
 
   initdata = extract_initial_data(result.body)
   if initdata.empty?
     error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip
     error_message ||= translate(locale, "Could not get channel info.")
-    raise error_message
+    raise InfoException.new(error_message)
   end
 
   author = about.xpath_node(%q(//meta[@name="title"])).not_nil!["content"]

src/invidious/comments.cr

@@ -92,7 +92,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
   response = JSON.parse(response.body)
 
   if !response["response"]["continuationContents"]?
-    raise translate(locale, "Could not fetch comments")
+    raise InfoException.new("Could not fetch comments")
   end
 
   response = response["response"]["continuationContents"]
@@ -266,7 +266,7 @@ def fetch_reddit_comments(id, sort_by = "confidence")
 
     thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink)
   else
-    raise "Got error code #{search_results.status_code}"
+    raise InfoException.new("Could not fetch comments")
   end
 
   comments = result[1].data.as(RedditListing).children

src/invidious/helpers/errors.cr

@@ -0,0 +1,90 @@
+# InfoExceptions are for displaying information to the user.
+#
+# An InfoException might or might not indicate that something went wrong.
+# Historically Invidious didn't differentiate between these two options, so to
+# maintain previous functionality InfoExceptions do not print backtraces.
+class InfoException < Exception
+end
+
+macro error_template(*args)
+  error_template_helper(env, config, locale, {{*args}})
+end
+
+def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
+  if exception.is_a?(InfoException)
+    return error_template_helper(env, config, locale, status_code, exception.message || "")
+  end
+  env.response.status_code = status_code
+  error_message = <<-END_HTML
+    Looks like you've found a bug in Invidious. Feel free to open a new issue
+    <a href="https://github.com/iv-org/invidious/issues">here</a>
+    or send an email to
+    <a href="mailto:#{CONFIG.admin_email}">#{CONFIG.admin_email}</a>.
+    <br>
+    <br>
+    <br>
+    Please include the following text in your message:
+    <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{exception.inspect_with_backtrace}</pre>
+  END_HTML
+  return templated "error"
+end
+
+def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
+  env.response.status_code = status_code
+  error_message = translate(locale, message)
+  return templated "error"
+end
+
+macro error_atom(*args)
+  error_atom_helper(env, config, locale, {{*args}})
+end
+
+def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
+  if exception.is_a?(InfoException)
+    return error_atom_helper(env, config, locale, status_code, exception.message || "")
+  end
+  env.response.content_type = "application/atom+xml"
+  env.response.status_code = status_code
+  return "<error>#{exception.inspect_with_backtrace}</error>"
+end
+
+def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
+  env.response.content_type = "application/atom+xml"
+  env.response.status_code = status_code
+  return "<error>#{message}</error>"
+end
+
+macro error_json(*args)
+  error_json_helper(env, config, locale, {{*args}})
+end
+
+def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil)
+  if exception.is_a?(InfoException)
+    return error_json_helper(env, config, locale, status_code, exception.message || "", additional_fields)
+  end
+  env.response.content_type = "application/json"
+  env.response.status_code = status_code
+  error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
+  if additional_fields
+    error_message = error_message.merge(additional_fields)
+  end
+  return error_message.to_json
+end
+
+def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception)
+  return error_json_helper(env, config, locale, status_code, exception, nil)
+end
+
+def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil)
+  env.response.content_type = "application/json"
+  env.response.status_code = status_code
+  error_message = {"error" => message}
+  if additional_fields
+    error_message = error_message.merge(additional_fields)
+  end
+  return error_message.to_json
+end
+
+def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String)
+  error_json_helper(env, config, locale, status_code, message, nil)
+end

src/invidious/helpers/tokens.cr

@@ -70,33 +70,33 @@ def validate_request(token, session, request, key, db, locale = nil)
   when JSON::Any
     token = token.as_h
   when Nil
-    raise translate(locale, "Hidden field \"token\" is a required field")
+    raise InfoException.new("Hidden field \"token\" is a required field")
   end
 
   expire = token["expire"]?.try &.as_i
   if expire.try &.< Time.utc.to_unix
-    raise translate(locale, "Token is expired, please try again")
+    raise InfoException.new("Token is expired, please try again")
   end
 
   if token["session"] != session
-    raise translate(locale, "Erroneous token")
+    raise InfoException.new("Erroneous token")
   end
 
   scopes = token["scopes"].as_a.map { |v| v.as_s }
   scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
   if !scopes_include_scope(scopes, scope)
-    raise translate(locale, "Invalid scope")
+    raise InfoException.new("Invalid scope")
   end
 
   if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
-    raise translate(locale, "Invalid signature")
+    raise InfoException.new("Invalid signature")
   end
 
   if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time}))
     if nonce[1] > Time.utc
       db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0])
     else
-      raise translate(locale, "Erroneous token")
+      raise InfoException.new("Erroneous token")
     end
   end
 

src/invidious/mixes.cr

@@ -30,7 +30,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
   initial_data = extract_initial_data(response.body)
 
   if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
-    raise translate(locale, "Could not create mix.")
+    raise InfoException.new("Could not create mix.")
   end
 
   playlist = initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"]

src/invidious/playlists.cr

@@ -338,7 +338,7 @@ def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
     if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
       return playlist
     else
-      raise "Playlist does not exist."
+      raise InfoException.new("Playlist does not exist.")
     end
   else
     return fetch_playlist(plid, locale)
@@ -353,16 +353,16 @@ def fetch_playlist(plid, locale)
   response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
   if response.status_code != 200
     if response.headers["location"]?.try &.includes? "/sorry/index"
-      raise "Could not extract playlist info. Instance is likely blocked."
+      raise InfoException.new("Could not extract playlist info. Instance is likely blocked.")
     else
-      raise translate(locale, "Not a playlist.")
+      raise InfoException.new("Not a playlist.")
     end
   end
 
   initial_data = extract_initial_data(response.body)
   playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]?
 
-  raise "Could not extract playlist info" if !playlist_info
+  raise InfoException.new("Could not extract playlist info") if !playlist_info
   title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
 
   desc_item = playlist_info["description"]?
@@ -390,7 +390,7 @@ def fetch_playlist(plid, locale)
   author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]?
     .try &.["videoOwner"]["videoOwnerRenderer"]?
 
-  raise "Could not extract author info" if !author_info
+  raise InfoException.new("Could not extract author info") if !author_info
 
   author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
   author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""

src/invidious/routes/embed/index.cr

@@ -8,9 +8,7 @@ class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute
         offset = env.params.query["index"]?.try &.to_i? || 0
         videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
       rescue ex
-        error_message = ex.message
-        env.response.status_code = 500
-        return templated "error"
+        return error_template(500, ex)
       end
 
       url = "/embed/#{videos[0].id}?#{env.params.query}"

src/invidious/routes/embed/show.cr

@@ -38,9 +38,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
           offset = env.params.query["index"]?.try &.to_i? || 0
           videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale)
         rescue ex
-          error_message = ex.message
-          env.response.status_code = 500
-          return templated "error"
+          return error_template(500, ex)
         end
 
         url = "/embed/#{videos[0].id}"
@@ -63,8 +61,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
       env.params.query.delete_all("channel")
 
       if !video_id || video_id == "live_stream"
-        error_message = "Video is unavailable."
-        return templated "error"
+        return error_template(500, "Video is unavailable.")
       end
 
       url = "/embed/#{video_id}"
@@ -100,9 +97,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute
     rescue ex : VideoRedirect
       return env.redirect env.request.resource.gsub(id, ex.video_id)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 500
-      return templated "error"
+      return error_template(500, ex)
     end
 
     if preferences.annotations_subscribed &&

src/invidious/routes/playlists.cr

@@ -56,26 +56,21 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     begin
       validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 400
-      return templated "error"
+      return error_template(400, ex)
     end
 
     title = env.params.body["title"]?.try &.as(String)
     if !title || title.empty?
-      error_message = "Title cannot be empty."
-      return templated "error"
+      return error_template(400, "Title cannot be empty.")
     end
 
     privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "")
     if !privacy
-      error_message = "Invalid privacy setting."
-      return templated "error"
+      return error_template(400, "Invalid privacy setting.")
     end
 
     if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100
-      error_message = "User cannot have more than 100 playlists."
-      return templated "error"
+      return error_template(400, "User cannot have more than 100 playlists.")
     end
 
     playlist = create_playlist(PG_DB, title, privacy, user)
@@ -142,9 +137,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     begin
       validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 400
-      return templated "error"
+      return error_template(400, ex)
     end
 
     playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
@@ -217,9 +210,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     begin
       validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 400
-      return templated "error"
+      return error_template(400, ex)
     end
 
     playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
@@ -306,9 +297,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
       if redirect
         return env.redirect referer
       else
-        error_message = {"error" => "No such user"}.to_json
-        env.response.status_code = 403
-        return error_message
+        return error_json(403, "No such user")
       end
     end
 
@@ -320,13 +309,9 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
       validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale)
     rescue ex
       if redirect
-        error_message = ex.message
-        env.response.status_code = 400
-        return templated "error"
+        return error_template(400, ex)
       else
-        error_message = {"error" => ex.message}.to_json
-        env.response.status_code = 400
-        return error_message
+        return error_json(400, ex)
       end
     end
 
@@ -353,13 +338,9 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
       raise "Invalid user" if playlist.author != user.email
     rescue ex
       if redirect
-        error_message = ex.message
-        env.response.status_code = 400
-        return templated "error"
+        return error_template(400, ex)
       else
-        error_message = {"error" => ex.message}.to_json
-        env.response.status_code = 400
-        return error_message
+        return error_json(400, ex)
       end
     end
 
@@ -374,13 +355,10 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
       # TODO: Playlist stub
     when "action_add_video"
       if playlist.index.size >= 500
-        env.response.status_code = 400
         if redirect
-          error_message = "Playlist cannot have more than 500 videos"
-          return templated "error"
+          return error_template(400, "Playlist cannot have more than 500 videos")
         else
-          error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json
-          return error_message
+          return error_json(400, "Playlist cannot have more than 500 videos")
         end
       end
 
@@ -389,13 +367,10 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
       begin
         video = get_video(video_id, PG_DB)
       rescue ex
-        env.response.status_code = 500
         if redirect
-          error_message = ex.message
-          return templated "error"
+          return error_template(500, ex)
         else
-          error_message = {"error" => ex.message}.to_json
-          return error_message
+          return error_json(500, ex)
         end
       end
 
@@ -423,9 +398,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     when "action_move_video_before"
       # TODO: Playlist stub
     else
-      error_message = {"error" => "Unsupported action #{action}"}.to_json
-      env.response.status_code = 400
-      return error_message
+      return error_json(400, "Unsupported action #{action}")
     end
 
     if redirect
@@ -457,15 +430,11 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     begin
       playlist = get_playlist(PG_DB, plid, locale)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 500
-      return templated "error"
+      return error_template(500, ex)
     end
 
     if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email
-      error_message = "This playlist is private."
-      env.response.status_code = 403
-      return templated "error"
+      return error_template(403, "This playlist is private.")
     end
 
     begin
@@ -495,9 +464,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute
     begin
       mix = fetch_mix(rdid, continuation, locale: locale)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 500
-      return templated "error"
+      return error_template(500, ex)
     end
 
     templated "mix"

src/invidious/routes/watch.cr

@@ -12,9 +12,7 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute
       id = env.params.query["v"]
 
       if env.params.query["v"].empty?
-        error_message = "Invalid parameters."
-        env.response.status_code = 400
-        return templated "error"
+        return error_template(400, "Invalid parameters.")
       end
 
       if id.size > 11
@@ -56,10 +54,8 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute
     rescue ex : VideoRedirect
       return env.redirect env.request.resource.gsub(id, ex.video_id)
     rescue ex
-      error_message = ex.message
-      env.response.status_code = 500
       logger.puts("#{id} : #{ex.message}")
-      return templated "error"
+      return error_template(500, ex)
     end
 
     if preferences.annotations_subscribed &&