add backup failure notification

backup.nix

@@ -16,6 +16,27 @@ let
   tarRedisStdoutCmd = pkgs.writeScript "backup-misskey-redis" ''
     tar -cz -C /var/lib/redis-misskey .
   '';
+
+  notifyError = pkgs.writeShellApplication {
+    name = "notify-error";
+
+    runtimeInputs = [ pkgs.sendmail ];
+
+    text = ''
+      from="$1"
+      subject="$2"
+      text="$3"
+
+      sendmail <<EOEMAIL
+      To: admin@heartles.xyz
+      From: noreply+$from@admin.egirls.gay
+      Subject: $subject
+
+      $text
+
+      EOEMAIL
+    '';
+  };
 in {
   users.users."${user}" = {
     isSystemUser = true;
@@ -76,6 +97,17 @@ in {
     after = [ "network-online.target" ];
     wants = [ "network-online.target" ];
     requires = [ "postgresql.service" ];
+
+    unitConfig.onFailure = ''
+      ${notifyError} "backup" "Misskey Backup Failure Notification" "$(<<EOMSG
+      A backup process has failed. Logs to follow:
+
+      $(journalctl _SYSTEMD_INVOCATION_ID=$(systemctl show --value -p InvocationID misskey-backup.service) -u misskey-backup.service)
+
+      EOMSG
+      )
+      "
+    '';
   };
 
   systemd.timers.misskey-backup = { timerConfig.Persistent = true; };