From 3aff383a2c6f1ce3b4e4ae402ff48649c1c01b3b Mon Sep 17 00:00:00 2001 From: jaina heartles Date: Wed, 25 Dec 2024 00:40:17 -0500 Subject: [PATCH 1/5] backup redis db and config dir --- backup.nix | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backup.nix b/backup.nix index cb11c1c..6950b39 100644 --- a/backup.nix +++ b/backup.nix @@ -21,13 +21,30 @@ let config.services.postgresql.package s3cmd coreutils + mktemp ]; excludeShellChecks = [ "SC1091" ]; text = '' source "${backupConfigFile}" - pg_dump misskey | gzip | s3cmd put --config "$S3CFG" - "s3://$MISSKEY_BACKUP_BUCKET/\$\{MISSKEY_BACKUP_PREFIX}misskey-pgdump-$(date --iso-8601).sql.gz" + + dir="$(mktemp --directory)" + echo "Using temp dir '$dir'" + + trap EXIT "rm -rf '$dir'" + + echo "Copying config" + cp /srv/misskey/.config "$dir/config" -r + + echo "Dumping postgres database..." + pg_dump misskey | gzip > "$dir/postgres.sql.gz" + + echo "Copying redis database..." + cp /var/lib/redis-misskey "$dir/redis" -r + + tar -cz -C "$dir" . | \ + s3cmd put --config "$S3CFG" - "s3://$MISSKEY_BACKUP_BUCKET/\$\{MISSKEY_BACKUP_PREFIX}misskey-$(date --iso-8601).tar.gz" ''; }; in { From 983f47440f35811eb5a21273603cd26711b83473 Mon Sep 17 00:00:00 2001 From: jaina heartles Date: Wed, 25 Dec 2024 15:28:15 -0500 Subject: [PATCH 2/5] use object storage directory instead of tar --- backup.nix | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/backup.nix b/backup.nix index 6950b39..8970461 100644 --- a/backup.nix +++ b/backup.nix @@ -10,9 +10,8 @@ let user = "misskey-backup"; group = user; - # shell script file to be sourced. must have values "MISSKEY_BACKUP_BUCKET" "MISSKEY_BACKUP_PREFIX" and "S3CFG" - # $S3CFG must be a path to a .s3cfg file compatible with s3cmd backupConfigFile = "/etc/misskey-backup/conf"; + s3Cfg = "/etc/misskey-backup/s3cfg"; backupScript = pkgs.writeShellApplication { name = "misskey-backup"; @@ -27,24 +26,22 @@ let excludeShellChecks = [ "SC1091" ]; text = '' - source "${backupConfigFile}" + bucket="$(cat "${backupConfigFile}" | grep 'bucket=' | sed 's/bucket \?= \?')" + prefix="$(cat "${backupConfigFile}" | grep 'prefix=' | sed 's/prefix \?= \?')" - dir="$(mktemp --directory)" - echo "Using temp dir '$dir'" + s3Dir="s3://$bucket/\$\{prefix}misskey-$(date --iso-8601)" + echo "Uploading backups to '$s3Dir'" - trap EXIT "rm -rf '$dir'" - - echo "Copying config" - cp /srv/misskey/.config "$dir/config" -r + echo "Uploading config" + tar -cz -C /srv/misskey/.config . | s3cmd put --config "${s3Cfg}" - "$s3Dir/config.tar.gz" echo "Dumping postgres database..." - pg_dump misskey | gzip > "$dir/postgres.sql.gz" + pg_dump misskey | gzip | s3cmd put --config "${s3Cfg}" - "$s3Dir/pg_dump.sql.gz" - echo "Copying redis database..." - cp /var/lib/redis-misskey "$dir/redis" -r + echo "Uploading redis database..." + tar -cz -C /var/lib/redis-misskey . | s3cmd put --config "${s3Cfg}" - "$s3Dir/redis.tar.gz" - tar -cz -C "$dir" . | \ - s3cmd put --config "$S3CFG" - "s3://$MISSKEY_BACKUP_BUCKET/\$\{MISSKEY_BACKUP_PREFIX}misskey-$(date --iso-8601).tar.gz" + echo "Backup complete to '$s3Dir'" ''; }; in { From 24094cad316f05085494e5233a619b339d1ebd9c Mon Sep 17 00:00:00 2001 From: jaina heartles Date: Wed, 25 Dec 2024 15:33:20 -0500 Subject: [PATCH 3/5] use bash function to upload --- backup.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backup.nix b/backup.nix index 8970461..b2ad438 100644 --- a/backup.nix +++ b/backup.nix @@ -32,14 +32,20 @@ let s3Dir="s3://$bucket/\$\{prefix}misskey-$(date --iso-8601)" echo "Uploading backups to '$s3Dir'" + function upload () { + name="$1" + + s3cmd put --config "${s3Cfg}" - "$s3Dir/$name" + } + echo "Uploading config" - tar -cz -C /srv/misskey/.config . | s3cmd put --config "${s3Cfg}" - "$s3Dir/config.tar.gz" + tar -cz -C /srv/misskey/.config . | upload "config.tar.gz" echo "Dumping postgres database..." - pg_dump misskey | gzip | s3cmd put --config "${s3Cfg}" - "$s3Dir/pg_dump.sql.gz" + pg_dump misskey | gzip | upload "pg_dump.sql.gz" echo "Uploading redis database..." - tar -cz -C /var/lib/redis-misskey . | s3cmd put --config "${s3Cfg}" - "$s3Dir/redis.tar.gz" + tar -cz -C /var/lib/redis-misskey . | upload "redis.tar.gz" echo "Backup complete to '$s3Dir'" ''; From 9606a755d0474abbd402bab7e1e0fc8be6e02359 Mon Sep 17 00:00:00 2001 From: jaina heartles Date: Wed, 25 Dec 2024 15:35:47 -0500 Subject: [PATCH 4/5] take config file via cmd line --- backup.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backup.nix b/backup.nix index b2ad438..f4fd03e 100644 --- a/backup.nix +++ b/backup.nix @@ -23,11 +23,12 @@ let mktemp ]; - excludeShellChecks = [ "SC1091" ]; - text = '' - bucket="$(cat "${backupConfigFile}" | grep 'bucket=' | sed 's/bucket \?= \?')" - prefix="$(cat "${backupConfigFile}" | grep 'prefix=' | sed 's/prefix \?= \?')" + configFile="$1" + s3cfg="$2" + + bucket="$(cat "$configFile" | grep 'bucket=' | sed 's/bucket \?= \?')" + prefix="$(cat "$configFile" | grep 'prefix=' | sed 's/prefix \?= \?')" s3Dir="s3://$bucket/\$\{prefix}misskey-$(date --iso-8601)" echo "Uploading backups to '$s3Dir'" @@ -35,7 +36,7 @@ let function upload () { name="$1" - s3cmd put --config "${s3Cfg}" - "$s3Dir/$name" + s3cmd put --config "$s3cfg" - "$s3Dir/$name" } echo "Uploading config" @@ -62,7 +63,7 @@ in { enable = true; systemCronJobs = [ # run every monday at ass in the morning, EST" - "0 8 0 0 1 ${user} ${backupScript}" + "0 8 0 0 1 ${user} ${backupScript} ${backupConfigFile} ${s3Cfg}" ]; }; } From aa193bf4232b827eeed32670e2469b5212ee7a91 Mon Sep 17 00:00:00 2001 From: jaina heartles Date: Wed, 25 Dec 2024 15:41:55 -0500 Subject: [PATCH 5/5] encrypt backups --- backup.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backup.nix b/backup.nix index f4fd03e..6f208da 100644 --- a/backup.nix +++ b/backup.nix @@ -20,13 +20,14 @@ let config.services.postgresql.package s3cmd coreutils - mktemp + age ]; text = '' configFile="$1" s3cfg="$2" + ageRecipient="age17ckyc69njpryytc63ynn545jswyucg28k5xg3043g3j6q38dxqwq0wzhm2" bucket="$(cat "$configFile" | grep 'bucket=' | sed 's/bucket \?= \?')" prefix="$(cat "$configFile" | grep 'prefix=' | sed 's/prefix \?= \?')" @@ -36,7 +37,7 @@ let function upload () { name="$1" - s3cmd put --config "$s3cfg" - "$s3Dir/$name" + age -r "$ageRecipient" | s3cmd put --config "$s3cfg" - "$s3Dir/$name.age" } echo "Uploading config"