ops: schedule home pilot backup and retain verified PC copies

deploy/pilot/backup.home-server.example

@@ -0,0 +1,4 @@
+# Test pilot on the owner's home server; no private key is stored here.
+BACKUP_DIRECTORY=/home/ochenstarik/krona-staff-pilot/backups
+BACKUP_RECIPIENT_CERT=/home/ochenstarik/krona-staff-pilot/deploy/pilot/secrets/backup-recipient.crt
+BACKUP_TRANSFER_MODE=external-pull

deploy/pilot/krona-staff-backup-user.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Encrypted Krona Staff PostgreSQL backup (user service)
+After=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=%h/krona-staff-pilot/deploy/pilot
+EnvironmentFile=%h/krona-staff-pilot/deploy/pilot/backup.env
+ExecStart=/bin/sh %h/krona-staff-pilot/deploy/pilot/backup.sh
+UMask=0077
+NoNewPrivileges=true

docs/BACKUP-RESTORE.md

@@ -31,6 +31,15 @@ disable host-key verification. A SHA-256 sidecar detects transfer corruption;
 it is not a digital signature. Other deployments may use push mode with an
 off-host SCP destination.
 
+The PC pull script keeps 90 days of local copies by default. Set
+`BACKUP_LOCAL_RETENTION_DAYS` to an integer from 1 to 36500 to change this period.
+After a successful pull and freshness check, it removes only direct child files with
+the generated `krona-staff-<UTC timestamp>-<pid>.dump.cms` name whose one-line
+SHA-256 sidecar names that same file and matches its bytes. It removes the matching
+`.sha256` sidecar with the encrypted file. Unverifiable pairs and symlinks are
+preserved, as are unrelated files and directories. This policy applies to PC copies;
+it does not prune the server's backup directory.
+
 The server script fails if dumping or encryption fails. In push mode it also fails
 if off-host transfer fails. It does not write an unencrypted dump to disk. In
 external-pull mode a successful server run reports `pending_pull`, never an
@@ -46,9 +55,8 @@ Monitor both the server service and the PC pull task exit statuses. Verify that
 the PC directory contains a recent copy; the server-only encrypted file is not
 evidence of off-host protection.
 
-Do not enable the schedule until the destination, recovery-key custody, retention
+Do not enable the schedule until the destination, recovery-key custody, server-side
-policy, and a successful restore drill have been agreed and tested. Retention
+retention policy, and a successful restore drill have been agreed and tested.
-cleanup is not implemented yet; the current script never prunes backups.
 
 ## Required restore drill before tenant migration
 
@@ -78,7 +86,6 @@ cleanup is not implemented yet; the current script never prunes backups.
 - Off-host storage and retention policy selected and configured.
 - Restore automation and repeatable reporting for later backups.
 - A fail-closed backup gate before migrations in `setup.sh`.
-- Retention implementation and failure-path tests.
 
 Until these items are complete, K18 remains open and pilot tenant migration is blocked.
 

scripts/pull-pilot-backups.sh

@@ -6,6 +6,12 @@ umask 077
 : "${PILOT_SSH_TARGET:?Set the configured SSH alias for the pilot server}"
 : "${PILOT_REMOTE_BACKUP_DIRECTORY:?Set its absolute encrypted backup directory}"
 : "${BACKUP_LOCAL_DIRECTORY:?Set an absolute backup directory on this PC}"
+BACKUP_LOCAL_RETENTION_DAYS=${BACKUP_LOCAL_RETENTION_DAYS:-90}
+[[ "$BACKUP_LOCAL_RETENTION_DAYS" =~ ^[1-9][0-9]{0,4}$ ]] &&
+  (( BACKUP_LOCAL_RETENTION_DAYS <= 36500 )) || {
+  echo 'BACKUP_LOCAL_RETENTION_DAYS must be an integer from 1 to 36500' >&2
+  exit 1
+}
 
 [[ "$PILOT_SSH_TARGET" =~ ^[a-zA-Z0-9_.@-]+$ ]] || { echo 'Invalid SSH target' >&2; exit 1; }
 [[ "$PILOT_REMOTE_BACKUP_DIRECTORY" =~ ^/[a-zA-Z0-9_./-]+$ ]] || { echo 'Invalid server backup directory' >&2; exit 1; }
@@ -73,4 +79,41 @@ now_epoch=$(date -u +%s)
   exit 1
 }
 
-printf 'off_host_backup_count=%s\nnewly_copied=%s\n' "$(find . -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' | wc -l)" "$copied"
+retention_cutoff=$((now_epoch - BACKUP_LOCAL_RETENTION_DAYS * 86400))
+pruned=0
+retention_skipped=0
+while IFS= read -r -d '' backup_path; do
+  name=${backup_path##*/}
+  [[ "$name" =~ ^krona-staff-([0-9]{8}T[0-9]{6}Z)-[0-9]+\.dump\.cms$ ]] || continue
+  timestamp=${BASH_REMATCH[1]}
+  backup_epoch=$(date -u -d "${timestamp:0:4}-${timestamp:4:2}-${timestamp:6:2} ${timestamp:9:2}:${timestamp:11:2}:${timestamp:13:2} UTC" +%s 2>/dev/null) || continue
+  [[ "$(date -u -d "@$backup_epoch" '+%Y%m%dT%H%M%SZ')" == "$timestamp" ]] || continue
+  (( backup_epoch < retention_cutoff )) || continue
+
+  checksum_path="$local_dir/$name.sha256"
+  if [[ ! -f "$backup_path" || -L "$backup_path" || ! -f "$checksum_path" || -L "$checksum_path" ]] ||
+    [[ "$(wc -l < "$checksum_path")" -ne 1 ]]; then
+    retention_skipped=$((retention_skipped + 1))
+    continue
+  fi
+  read -r expected stored_name extra < "$checksum_path" || {
+    retention_skipped=$((retention_skipped + 1))
+    continue
+  }
+  if [[ ! "$expected" =~ ^[[:xdigit:]]{64}$ || "$stored_name" != "$name" || -n "${extra:-}" ]]; then
+    retention_skipped=$((retention_skipped + 1))
+    continue
+  fi
+  actual=$(sha256sum -- "$backup_path")
+  if [[ "${actual%% *}" != "$expected" ]]; then
+    retention_skipped=$((retention_skipped + 1))
+    continue
+  fi
+
+  rm -- "$backup_path" "$checksum_path"
+  pruned=$((pruned + 1))
+done < <(find "$local_dir" -mindepth 1 -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' -print0)
+
+printf 'off_host_backup_count=%s\nnewly_copied=%s\nretention_pruned=%s\nretention_skipped=%s\n' \
+  "$(find "$local_dir" -maxdepth 1 -type f -name 'krona-staff-*.dump.cms' | wc -l)" \
+  "$copied" "$pruned" "$retention_skipped"

scripts/run-pilot-backup-pull.ps1

@@ -0,0 +1,23 @@
+$ErrorActionPreference = 'Stop'
+
+$env:PILOT_SSH_TARGET = 'krona-home'
+$env:PILOT_REMOTE_BACKUP_DIRECTORY = '/home/ochenstarik/krona-staff-pilot/backups'
+$env:BACKUP_LOCAL_DIRECTORY = '/c/Users/trush/Documents/KronaStaffBackups'
+$env:BACKUP_MAX_AGE_HOURS = '36'
+
+$taskBash = 'C:\Program Files\Git\bin\bash.exe'
+$taskPull = Join-Path $PSScriptRoot 'pull-pilot-backups.sh'
+$taskLog = 'C:\Users\trush\Documents\KronaStaffBackups\pull.log'
+
+try {
+  $taskResult = & $taskBash $taskPull 2>&1
+  $taskExit = $LASTEXITCODE
+  $taskResult | Out-File -LiteralPath $taskLog -Append -Encoding utf8
+  if ($taskExit -ne 0) {
+    throw "Backup pull failed with exit code $taskExit"
+  }
+} catch {
+  "$(Get-Date -Format o) $($_.Exception.Message)" |
+    Out-File -LiteralPath $taskLog -Append -Encoding utf8
+  exit 1
+}

scripts/test-backup-pull.sh

@@ -68,6 +68,7 @@ run_pull() {
     PILOT_REMOTE_BACKUP_DIRECTORY="$remote_dir" \
     BACKUP_LOCAL_DIRECTORY="$local_dir" \
     BACKUP_MAX_AGE_HOURS=36 \
+    BACKUP_LOCAL_RETENTION_DAYS=90 \
     MOCK_REMOTE_STORE="$remote_store" \
     MOCK_REMOTE_LISTING="$listing" \
     MOCK_SCP_STATE="$state" \
@@ -90,8 +91,9 @@ make_backup() {
 
 assert_rejected_before_ssh() {
   local label=$1 remote_dir=$2 local_dir=$3 state=$4
+  shift 4
   rm -f -- "$ssh_log"
-  if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" > "$test_root/reject.log" 2>&1; then
+  if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" "$@" > "$test_root/reject.log" 2>&1; then
     record_failure "$label unexpectedly succeeded"
   elif [[ -e "$ssh_log" ]]; then
     record_failure "$label reached mocked ssh before rejection"
@@ -121,14 +123,62 @@ second_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$su
 [[ "$second_output" == *'newly_copied=0'* ]] || record_failure 'repeat pull was not idempotent'
 [[ "$(cat "$success_state")" == 2 ]] || record_failure 'repeat pull issued additional scp calls'
 
+old_stamp=$(date -u -d '91 days ago' '+%Y%m%dT%H%M%SZ')
+old_name="krona-staff-$old_stamp-201.dump.cms"
+recent_stamp=$(date -u -d '10 days ago' '+%Y%m%dT%H%M%SZ')
+recent_name="krona-staff-$recent_stamp-202.dump.cms"
+unverified_name="krona-staff-$old_stamp-203.dump.cms"
+symlink_name="krona-staff-$old_stamp-204.dump.cms"
+make_backup "$old_name"
+make_backup "$recent_name"
+make_backup "$unverified_name"
+make_backup "$symlink_name"
+retention_local="$test_root/local-retention"
+mkdir -p "$retention_local"
+for name in "$old_name" "$recent_name" "$unverified_name"; do
+  cp -- "$remote_store/$name" "$retention_local/$name"
+  cp -- "$remote_store/$name.sha256" "$retention_local/$name.sha256"
+done
+printf 'tampered synthetic bytes\n' >> "$retention_local/$unverified_name"
+printf 'preserve unrelated file\n' > "$retention_local/operator-notes.txt"
+printf 'symlink target\n' > "$test_root/symlink-target.bin"
+ln -s "$test_root/symlink-target.bin" "$retention_local/$symlink_name"
+cp -- "$remote_store/$symlink_name.sha256" "$retention_local/$symlink_name.sha256"
+retention_output=$(run_pull "$remote_directory" "$retention_local" "$fresh_name" "$test_root/scp-retention.count" BACKUP_LOCAL_RETENTION_DAYS=90)
+[[ ! -e "$retention_local/$old_name" && ! -e "$retention_local/$old_name.sha256" ]] ||
+  record_failure 'verified backup older than 90 days was not pruned with its checksum'
+[[ -f "$retention_local/$recent_name" && -f "$retention_local/$recent_name.sha256" ]] ||
+  record_failure 'recent backup was pruned'
+[[ -f "$retention_local/$unverified_name" && -f "$retention_local/$unverified_name.sha256" ]] ||
+  record_failure 'unverified old backup was removed'
+[[ -L "$retention_local/$symlink_name" && -f "$test_root/symlink-target.bin" ]] ||
+  record_failure 'symlink backup or its target was changed'
+[[ -f "$retention_local/$symlink_name.sha256" && -f "$retention_local/operator-notes.txt" ]] ||
+  record_failure 'retention removed a symlink sidecar or unrelated file'
+[[ "$retention_output" == *'retention_pruned=1'* ]] || record_failure 'retention did not report one pruned backup'
+[[ "$retention_output" == *'retention_skipped=1'* ]] || record_failure 'retention did not report the unverified backup'
+
+assert_rejected_before_ssh 'zero retention' "$remote_directory" "$test_root/local-invalid-retention" "$test_root/scp-invalid-retention.count" BACKUP_LOCAL_RETENTION_DAYS=0
+grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" ||
+  record_failure 'invalid retention failed for an unexpected reason'
+assert_rejected_before_ssh 'excessive retention' "$remote_directory" "$test_root/local-excessive-retention" "$test_root/scp-excessive-retention.count" BACKUP_LOCAL_RETENTION_DAYS=36501
+grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" ||
+  record_failure 'excessive retention failed for an unexpected reason'
+
 stale_stamp=$(date -u -d '48 hours ago' '+%Y%m%dT%H%M%SZ')
 stale_name="krona-staff-$stale_stamp-102.dump.cms"
 make_backup "$stale_name"
-if run_pull "$remote_directory" "$test_root/local-stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then
+local_stale="$test_root/local-stale"
+mkdir -p "$local_stale"
+cp -- "$remote_store/$old_name" "$local_stale/$old_name"
+cp -- "$remote_store/$old_name.sha256" "$local_stale/$old_name.sha256"
+if run_pull "$remote_directory" "$local_stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then
   record_failure 'stale backup unexpectedly succeeded'
 elif ! grep -q 'Newest encrypted backup is stale or has a future timestamp' "$test_root/stale.log"; then
   record_failure 'stale backup failed for an unexpected reason'
 fi
+[[ -f "$local_stale/$old_name" && -f "$local_stale/$old_name.sha256" ]] ||
+  record_failure 'failed stale pull pruned an existing local backup'
 
 bad_checksum_name="krona-staff-$fresh_stamp-103.dump.cms"
 make_backup "$bad_checksum_name"
@@ -145,6 +195,9 @@ assert_rejected_before_ssh 'remote traversal' /srv/krona-staff/../escape/ "$test
 assert_rejected_before_ssh 'local root' "$remote_directory" / "$test_root/scp-local-root.count"
 local_traversal="$test_root/local-traversal/../local-target"
 assert_rejected_before_ssh 'local traversal' "$remote_directory" "$local_traversal" "$test_root/scp-local-traversal.count"
+local_directory_symlink="$test_root/local-directory-symlink"
+ln -s "$local_success" "$local_directory_symlink"
+assert_rejected_before_ssh 'local directory symlink' "$remote_directory" "$local_directory_symlink" "$test_root/scp-local-symlink.count"
 
 scp_failure_name="krona-staff-$fresh_stamp-104.dump.cms"
 make_backup "$scp_failure_name"