From 4fc053850f9a2c9a8d5d3ab179744419310c716c Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sat, 26 Sep 2026 17:07:36 +0700 Subject: [PATCH] ops: schedule home pilot backup and retain verified PC copies --- deploy/pilot/backup.home-server.example | 4 ++ deploy/pilot/krona-staff-backup-user.service | 11 ++++ docs/BACKUP-RESTORE.md | 15 ++++-- scripts/pull-pilot-backups.sh | 45 +++++++++++++++- scripts/run-pilot-backup-pull.ps1 | 23 ++++++++ scripts/test-backup-pull.sh | 57 +++++++++++++++++++- 6 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 deploy/pilot/backup.home-server.example create mode 100644 deploy/pilot/krona-staff-backup-user.service create mode 100644 scripts/run-pilot-backup-pull.ps1 diff --git a/deploy/pilot/backup.home-server.example b/deploy/pilot/backup.home-server.example new file mode 100644 index 0000000..fddbf32 --- /dev/null +++ b/deploy/pilot/backup.home-server.example @@ -0,0 +1,4 @@ +# Test pilot on the owner's home server; no private key is stored here. +BACKUP_DIRECTORY=/home/ochenstarik/krona-staff-pilot/backups +BACKUP_RECIPIENT_CERT=/home/ochenstarik/krona-staff-pilot/deploy/pilot/secrets/backup-recipient.crt +BACKUP_TRANSFER_MODE=external-pull diff --git a/deploy/pilot/krona-staff-backup-user.service b/deploy/pilot/krona-staff-backup-user.service new file mode 100644 index 0000000..1f29af7 --- /dev/null +++ b/deploy/pilot/krona-staff-backup-user.service @@ -0,0 +1,11 @@ +[Unit] +Description=Encrypted Krona Staff PostgreSQL backup (user service) +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=%h/krona-staff-pilot/deploy/pilot +EnvironmentFile=%h/krona-staff-pilot/deploy/pilot/backup.env +ExecStart=/bin/sh %h/krona-staff-pilot/deploy/pilot/backup.sh +UMask=0077 +NoNewPrivileges=true diff --git a/docs/BACKUP-RESTORE.md b/docs/BACKUP-RESTORE.md index 7960d20..764ff49 100644 --- a/docs/BACKUP-RESTORE.md +++ b/docs/BACKUP-RESTORE.md @@ -31,6 +31,15 @@ disable host-key verification. A SHA-256 sidecar detects transfer corruption; it is not a digital signature. Other deployments may use push mode with an off-host SCP destination. +The PC pull script keeps 90 days of local copies by default. Set +`BACKUP_LOCAL_RETENTION_DAYS` to an integer from 1 to 36500 to change this period. +After a successful pull and freshness check, it removes only direct child files with +the generated `krona-staff--.dump.cms` name whose one-line +SHA-256 sidecar names that same file and matches its bytes. It removes the matching +`.sha256` sidecar with the encrypted file. Unverifiable pairs and symlinks are +preserved, as are unrelated files and directories. This policy applies to PC copies; +it does not prune the server's backup directory. + The server script fails if dumping or encryption fails. In push mode it also fails if off-host transfer fails. It does not write an unencrypted dump to disk. In external-pull mode a successful server run reports `pending_pull`, never an @@ -46,9 +55,8 @@ Monitor both the server service and the PC pull task exit statuses. Verify that the PC directory contains a recent copy; the server-only encrypted file is not evidence of off-host protection. -Do not enable the schedule until the destination, recovery-key custody, retention -policy, and a successful restore drill have been agreed and tested. Retention -cleanup is not implemented yet; the current script never prunes backups. +Do not enable the schedule until the destination, recovery-key custody, server-side +retention policy, and a successful restore drill have been agreed and tested. ## Required restore drill before tenant migration @@ -78,7 +86,6 @@ cleanup is not implemented yet; the current script never prunes backups. - Off-host storage and retention policy selected and configured. - Restore automation and repeatable reporting for later backups. - A fail-closed backup gate before migrations in `setup.sh`. -- Retention implementation and failure-path tests. Until these items are complete, K18 remains open and pilot tenant migration is blocked. diff --git a/scripts/pull-pilot-backups.sh b/scripts/pull-pilot-backups.sh index 9fe09e1..acbea95 100644 --- a/scripts/pull-pilot-backups.sh +++ b/scripts/pull-pilot-backups.sh @@ -6,6 +6,12 @@ umask 077 : "${PILOT_SSH_TARGET:?Set the configured SSH alias for the pilot server}" : "${PILOT_REMOTE_BACKUP_DIRECTORY:?Set its absolute encrypted backup directory}" : "${BACKUP_LOCAL_DIRECTORY:?Set an absolute backup directory on this PC}" +BACKUP_LOCAL_RETENTION_DAYS=${BACKUP_LOCAL_RETENTION_DAYS:-90} +[[ "$BACKUP_LOCAL_RETENTION_DAYS" =~ ^[1-9][0-9]{0,4}$ ]] && + (( BACKUP_LOCAL_RETENTION_DAYS <= 36500 )) || { + echo 'BACKUP_LOCAL_RETENTION_DAYS must be an integer from 1 to 36500' >&2 + exit 1 +} [[ "$PILOT_SSH_TARGET" =~ ^[a-zA-Z0-9_.@-]+$ ]] || { echo 'Invalid SSH target' >&2; exit 1; } [[ "$PILOT_REMOTE_BACKUP_DIRECTORY" =~ ^/[a-zA-Z0-9_./-]+$ ]] || { echo 'Invalid server backup directory' >&2; exit 1; } @@ -73,4 +79,41 @@ now_epoch=$(date -u +%s) exit 1 } -printf 'off_host_backup_count=%s\nnewly_copied=%s\n' "$(find . -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' | wc -l)" "$copied" +retention_cutoff=$((now_epoch - BACKUP_LOCAL_RETENTION_DAYS * 86400)) +pruned=0 +retention_skipped=0 +while IFS= read -r -d '' backup_path; do + name=${backup_path##*/} + [[ "$name" =~ ^krona-staff-([0-9]{8}T[0-9]{6}Z)-[0-9]+\.dump\.cms$ ]] || continue + timestamp=${BASH_REMATCH[1]} + backup_epoch=$(date -u -d "${timestamp:0:4}-${timestamp:4:2}-${timestamp:6:2} ${timestamp:9:2}:${timestamp:11:2}:${timestamp:13:2} UTC" +%s 2>/dev/null) || continue + [[ "$(date -u -d "@$backup_epoch" '+%Y%m%dT%H%M%SZ')" == "$timestamp" ]] || continue + (( backup_epoch < retention_cutoff )) || continue + + checksum_path="$local_dir/$name.sha256" + if [[ ! -f "$backup_path" || -L "$backup_path" || ! -f "$checksum_path" || -L "$checksum_path" ]] || + [[ "$(wc -l < "$checksum_path")" -ne 1 ]]; then + retention_skipped=$((retention_skipped + 1)) + continue + fi + read -r expected stored_name extra < "$checksum_path" || { + retention_skipped=$((retention_skipped + 1)) + continue + } + if [[ ! "$expected" =~ ^[[:xdigit:]]{64}$ || "$stored_name" != "$name" || -n "${extra:-}" ]]; then + retention_skipped=$((retention_skipped + 1)) + continue + fi + actual=$(sha256sum -- "$backup_path") + if [[ "${actual%% *}" != "$expected" ]]; then + retention_skipped=$((retention_skipped + 1)) + continue + fi + + rm -- "$backup_path" "$checksum_path" + pruned=$((pruned + 1)) +done < <(find "$local_dir" -mindepth 1 -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' -print0) + +printf 'off_host_backup_count=%s\nnewly_copied=%s\nretention_pruned=%s\nretention_skipped=%s\n' \ + "$(find "$local_dir" -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' | wc -l)" \ + "$copied" "$pruned" "$retention_skipped" diff --git a/scripts/run-pilot-backup-pull.ps1 b/scripts/run-pilot-backup-pull.ps1 new file mode 100644 index 0000000..6e9367f --- /dev/null +++ b/scripts/run-pilot-backup-pull.ps1 @@ -0,0 +1,23 @@ +$ErrorActionPreference = 'Stop' + +$env:PILOT_SSH_TARGET = 'krona-home' +$env:PILOT_REMOTE_BACKUP_DIRECTORY = '/home/ochenstarik/krona-staff-pilot/backups' +$env:BACKUP_LOCAL_DIRECTORY = '/c/Users/trush/Documents/KronaStaffBackups' +$env:BACKUP_MAX_AGE_HOURS = '36' + +$taskBash = 'C:\Program Files\Git\bin\bash.exe' +$taskPull = Join-Path $PSScriptRoot 'pull-pilot-backups.sh' +$taskLog = 'C:\Users\trush\Documents\KronaStaffBackups\pull.log' + +try { + $taskResult = & $taskBash $taskPull 2>&1 + $taskExit = $LASTEXITCODE + $taskResult | Out-File -LiteralPath $taskLog -Append -Encoding utf8 + if ($taskExit -ne 0) { + throw "Backup pull failed with exit code $taskExit" + } +} catch { + "$(Get-Date -Format o) $($_.Exception.Message)" | + Out-File -LiteralPath $taskLog -Append -Encoding utf8 + exit 1 +} diff --git a/scripts/test-backup-pull.sh b/scripts/test-backup-pull.sh index aa03e8d..d650bc1 100644 --- a/scripts/test-backup-pull.sh +++ b/scripts/test-backup-pull.sh @@ -68,6 +68,7 @@ run_pull() { PILOT_REMOTE_BACKUP_DIRECTORY="$remote_dir" \ BACKUP_LOCAL_DIRECTORY="$local_dir" \ BACKUP_MAX_AGE_HOURS=36 \ + BACKUP_LOCAL_RETENTION_DAYS=90 \ MOCK_REMOTE_STORE="$remote_store" \ MOCK_REMOTE_LISTING="$listing" \ MOCK_SCP_STATE="$state" \ @@ -90,8 +91,9 @@ make_backup() { assert_rejected_before_ssh() { local label=$1 remote_dir=$2 local_dir=$3 state=$4 + shift 4 rm -f -- "$ssh_log" - if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" > "$test_root/reject.log" 2>&1; then + if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" "$@" > "$test_root/reject.log" 2>&1; then record_failure "$label unexpectedly succeeded" elif [[ -e "$ssh_log" ]]; then record_failure "$label reached mocked ssh before rejection" @@ -121,14 +123,62 @@ second_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$su [[ "$second_output" == *'newly_copied=0'* ]] || record_failure 'repeat pull was not idempotent' [[ "$(cat "$success_state")" == 2 ]] || record_failure 'repeat pull issued additional scp calls' +old_stamp=$(date -u -d '91 days ago' '+%Y%m%dT%H%M%SZ') +old_name="krona-staff-$old_stamp-201.dump.cms" +recent_stamp=$(date -u -d '10 days ago' '+%Y%m%dT%H%M%SZ') +recent_name="krona-staff-$recent_stamp-202.dump.cms" +unverified_name="krona-staff-$old_stamp-203.dump.cms" +symlink_name="krona-staff-$old_stamp-204.dump.cms" +make_backup "$old_name" +make_backup "$recent_name" +make_backup "$unverified_name" +make_backup "$symlink_name" +retention_local="$test_root/local-retention" +mkdir -p "$retention_local" +for name in "$old_name" "$recent_name" "$unverified_name"; do + cp -- "$remote_store/$name" "$retention_local/$name" + cp -- "$remote_store/$name.sha256" "$retention_local/$name.sha256" +done +printf 'tampered synthetic bytes\n' >> "$retention_local/$unverified_name" +printf 'preserve unrelated file\n' > "$retention_local/operator-notes.txt" +printf 'symlink target\n' > "$test_root/symlink-target.bin" +ln -s "$test_root/symlink-target.bin" "$retention_local/$symlink_name" +cp -- "$remote_store/$symlink_name.sha256" "$retention_local/$symlink_name.sha256" +retention_output=$(run_pull "$remote_directory" "$retention_local" "$fresh_name" "$test_root/scp-retention.count" BACKUP_LOCAL_RETENTION_DAYS=90) +[[ ! -e "$retention_local/$old_name" && ! -e "$retention_local/$old_name.sha256" ]] || + record_failure 'verified backup older than 90 days was not pruned with its checksum' +[[ -f "$retention_local/$recent_name" && -f "$retention_local/$recent_name.sha256" ]] || + record_failure 'recent backup was pruned' +[[ -f "$retention_local/$unverified_name" && -f "$retention_local/$unverified_name.sha256" ]] || + record_failure 'unverified old backup was removed' +[[ -L "$retention_local/$symlink_name" && -f "$test_root/symlink-target.bin" ]] || + record_failure 'symlink backup or its target was changed' +[[ -f "$retention_local/$symlink_name.sha256" && -f "$retention_local/operator-notes.txt" ]] || + record_failure 'retention removed a symlink sidecar or unrelated file' +[[ "$retention_output" == *'retention_pruned=1'* ]] || record_failure 'retention did not report one pruned backup' +[[ "$retention_output" == *'retention_skipped=1'* ]] || record_failure 'retention did not report the unverified backup' + +assert_rejected_before_ssh 'zero retention' "$remote_directory" "$test_root/local-invalid-retention" "$test_root/scp-invalid-retention.count" BACKUP_LOCAL_RETENTION_DAYS=0 +grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" || + record_failure 'invalid retention failed for an unexpected reason' +assert_rejected_before_ssh 'excessive retention' "$remote_directory" "$test_root/local-excessive-retention" "$test_root/scp-excessive-retention.count" BACKUP_LOCAL_RETENTION_DAYS=36501 +grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" || + record_failure 'excessive retention failed for an unexpected reason' + stale_stamp=$(date -u -d '48 hours ago' '+%Y%m%dT%H%M%SZ') stale_name="krona-staff-$stale_stamp-102.dump.cms" make_backup "$stale_name" -if run_pull "$remote_directory" "$test_root/local-stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then +local_stale="$test_root/local-stale" +mkdir -p "$local_stale" +cp -- "$remote_store/$old_name" "$local_stale/$old_name" +cp -- "$remote_store/$old_name.sha256" "$local_stale/$old_name.sha256" +if run_pull "$remote_directory" "$local_stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then record_failure 'stale backup unexpectedly succeeded' elif ! grep -q 'Newest encrypted backup is stale or has a future timestamp' "$test_root/stale.log"; then record_failure 'stale backup failed for an unexpected reason' fi +[[ -f "$local_stale/$old_name" && -f "$local_stale/$old_name.sha256" ]] || + record_failure 'failed stale pull pruned an existing local backup' bad_checksum_name="krona-staff-$fresh_stamp-103.dump.cms" make_backup "$bad_checksum_name" @@ -145,6 +195,9 @@ assert_rejected_before_ssh 'remote traversal' /srv/krona-staff/../escape/ "$test assert_rejected_before_ssh 'local root' "$remote_directory" / "$test_root/scp-local-root.count" local_traversal="$test_root/local-traversal/../local-target" assert_rejected_before_ssh 'local traversal' "$remote_directory" "$local_traversal" "$test_root/scp-local-traversal.count" +local_directory_symlink="$test_root/local-directory-symlink" +ln -s "$local_success" "$local_directory_symlink" +assert_rejected_before_ssh 'local directory symlink' "$remote_directory" "$local_directory_symlink" "$test_root/scp-local-symlink.count" scp_failure_name="krona-staff-$fresh_stamp-104.dump.cms" make_backup "$scp_failure_name"