#!/usr/bin/env bash set -euo pipefail repo_root=$(cd -- "$(dirname -- "$0")/.." && pwd -P) pull_script="$repo_root/scripts/pull-pilot-backups.sh" test_parent=$(cd -- /tmp && pwd -P) test_root=$(mktemp -d "$test_parent/krona-backup-pull-test.XXXXXX") test_root=$(cd -- "$test_root" && pwd -P) is_test_root() { [ "$(dirname -- "$1")" = "$test_parent" ] || return 1 case "$(basename -- "$1")" in krona-backup-pull-test.*) return 0 ;; *) return 1 ;; esac } if ! is_test_root "$test_root"; then printf 'unexpected temporary test directory: %s\n' "$test_root" >&2 exit 1 fi mock_bin="$test_root/bin" remote_store="$test_root/mock-remote" mkdir -p "$mock_bin" "$remote_store" cleanup() { resolved_test_root=$(cd -- "$test_root" && pwd -P) || return 1 if ! is_test_root "$resolved_test_root"; then printf 'refusing to remove unexpected test directory: %s\n' "$resolved_test_root" >&2 return 1 fi rm -rf -- "$resolved_test_root" } trap cleanup EXIT cat > "$mock_bin/ssh" <<'MOCK_SSH' #!/usr/bin/env bash set -euo pipefail [[ "$#" -eq 5 && "$1" == -o && "$2" == BatchMode=yes && "$3" == -- ]] || exit 61 printf '%s\n' "$4" "$5" >> "$MOCK_SSH_LOG" printf '%s' "$MOCK_REMOTE_LISTING" MOCK_SSH cat > "$mock_bin/scp" <<'MOCK_SCP' #!/usr/bin/env bash set -euo pipefail [[ "$#" -eq 4 && "$1" == -B && "$2" == -- ]] || exit 62 count=0 [[ ! -f "$MOCK_SCP_STATE" ]] || read -r count < "$MOCK_SCP_STATE" count=$((count + 1)) printf '%s\n' "$count" > "$MOCK_SCP_STATE" if [[ "${MOCK_SCP_FAIL_AT:-0}" == "$count" ]]; then exit 63; fi source_name=${3##*/} cp -- "$MOCK_REMOTE_STORE/$source_name" "$4" MOCK_SCP chmod +x "$mock_bin/ssh" "$mock_bin/scp" export PATH="$mock_bin:$PATH" cert="$test_root/recipient.crt" key="$test_root/recipient.key" /usr/bin/openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 1 \ -keyout "$key" -out "$cert" -subj '/CN=pull-test-recipient' >/dev/null 2>&1 remote_directory=/srv/krona-staff/backups/ ssh_log="$test_root/ssh.log" success_state="$test_root/scp-success.count" assertion_failures=0 run_pull() { local remote_dir=$1 local_dir=$2 listing=$3 state=$4 shift 4 env PATH="$mock_bin:$PATH" \ PILOT_SSH_TARGET=pull-test@invalid \ PILOT_REMOTE_BACKUP_DIRECTORY="$remote_dir" \ BACKUP_LOCAL_DIRECTORY="$local_dir" \ BACKUP_MAX_AGE_HOURS=36 \ BACKUP_LOCAL_RETENTION_DAYS=90 \ MOCK_REMOTE_STORE="$remote_store" \ MOCK_REMOTE_LISTING="$listing" \ MOCK_SCP_STATE="$state" \ MOCK_SSH_LOG="$ssh_log" \ "$@" bash "$pull_script" } record_failure() { printf 'FAIL: %s\n' "$*" >&2 assertion_failures=$((assertion_failures + 1)) } make_backup() { local name=$1 printf 'synthetic PostgreSQL backup\n' | /usr/bin/openssl cms -encrypt -binary \ -aes-256-cbc -outform DER -stream -in /dev/stdin \ -out "$remote_store/$name" "$cert" (cd -- "$remote_store" && sha256sum -- "$name" > "$name.sha256") } assert_rejected_before_ssh() { local label=$1 remote_dir=$2 local_dir=$3 state=$4 shift 4 rm -f -- "$ssh_log" if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" "$@" > "$test_root/reject.log" 2>&1; then record_failure "$label unexpectedly succeeded" elif [[ -e "$ssh_log" ]]; then record_failure "$label reached mocked ssh before rejection" fi } [[ "$(command -v ssh)" == "$mock_bin/ssh" ]] || { echo 'mock ssh is not first in PATH' >&2; exit 1; } [[ "$(command -v scp)" == "$mock_bin/scp" ]] || { echo 'mock scp is not first in PATH' >&2; exit 1; } bash -n "$pull_script" fresh_stamp=$(date -u '+%Y%m%dT%H%M%SZ') fresh_name="krona-staff-$fresh_stamp-101.dump.cms" make_backup "$fresh_name" local_success="$test_root/local-success" first_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$success_state") [[ -f "$local_success/$fresh_name" && -f "$local_success/$fresh_name.sha256" ]] || { record_failure 'fresh CMS and checksum were not installed locally' } cmp -s "$remote_store/$fresh_name" "$local_success/$fresh_name" || record_failure 'fresh CMS differs from remote bytes' (cd -- "$local_success" && sha256sum -c -- "$fresh_name.sha256") >/dev/null || { record_failure 'fresh local checksum does not validate' } [[ "$first_output" == *'newly_copied=1'* ]] || record_failure 'first pull did not report one new copy' [[ "$(cat "$success_state")" == 2 ]] || record_failure 'first pull did not fetch exactly CMS and checksum' second_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$success_state") [[ "$second_output" == *'newly_copied=0'* ]] || record_failure 'repeat pull was not idempotent' [[ "$(cat "$success_state")" == 2 ]] || record_failure 'repeat pull issued additional scp calls' old_stamp=$(date -u -d '91 days ago' '+%Y%m%dT%H%M%SZ') old_name="krona-staff-$old_stamp-201.dump.cms" recent_stamp=$(date -u -d '10 days ago' '+%Y%m%dT%H%M%SZ') recent_name="krona-staff-$recent_stamp-202.dump.cms" unverified_name="krona-staff-$old_stamp-203.dump.cms" symlink_name="krona-staff-$old_stamp-204.dump.cms" make_backup "$old_name" make_backup "$recent_name" make_backup "$unverified_name" make_backup "$symlink_name" retention_local="$test_root/local-retention" mkdir -p "$retention_local" for name in "$old_name" "$recent_name" "$unverified_name"; do cp -- "$remote_store/$name" "$retention_local/$name" cp -- "$remote_store/$name.sha256" "$retention_local/$name.sha256" done printf 'tampered synthetic bytes\n' >> "$retention_local/$unverified_name" printf 'preserve unrelated file\n' > "$retention_local/operator-notes.txt" printf 'symlink target\n' > "$test_root/symlink-target.bin" ln -s "$test_root/symlink-target.bin" "$retention_local/$symlink_name" cp -- "$remote_store/$symlink_name.sha256" "$retention_local/$symlink_name.sha256" retention_output=$(run_pull "$remote_directory" "$retention_local" "$fresh_name" "$test_root/scp-retention.count" BACKUP_LOCAL_RETENTION_DAYS=90) [[ ! -e "$retention_local/$old_name" && ! -e "$retention_local/$old_name.sha256" ]] || record_failure 'verified backup older than 90 days was not pruned with its checksum' [[ -f "$retention_local/$recent_name" && -f "$retention_local/$recent_name.sha256" ]] || record_failure 'recent backup was pruned' [[ -f "$retention_local/$unverified_name" && -f "$retention_local/$unverified_name.sha256" ]] || record_failure 'unverified old backup was removed' [[ -L "$retention_local/$symlink_name" && -f "$test_root/symlink-target.bin" ]] || record_failure 'symlink backup or its target was changed' [[ -f "$retention_local/$symlink_name.sha256" && -f "$retention_local/operator-notes.txt" ]] || record_failure 'retention removed a symlink sidecar or unrelated file' [[ "$retention_output" == *'retention_pruned=1'* ]] || record_failure 'retention did not report one pruned backup' [[ "$retention_output" == *'retention_skipped=1'* ]] || record_failure 'retention did not report the unverified backup' assert_rejected_before_ssh 'zero retention' "$remote_directory" "$test_root/local-invalid-retention" "$test_root/scp-invalid-retention.count" BACKUP_LOCAL_RETENTION_DAYS=0 grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" || record_failure 'invalid retention failed for an unexpected reason' assert_rejected_before_ssh 'excessive retention' "$remote_directory" "$test_root/local-excessive-retention" "$test_root/scp-excessive-retention.count" BACKUP_LOCAL_RETENTION_DAYS=36501 grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" || record_failure 'excessive retention failed for an unexpected reason' stale_stamp=$(date -u -d '48 hours ago' '+%Y%m%dT%H%M%SZ') stale_name="krona-staff-$stale_stamp-102.dump.cms" make_backup "$stale_name" local_stale="$test_root/local-stale" mkdir -p "$local_stale" cp -- "$remote_store/$old_name" "$local_stale/$old_name" cp -- "$remote_store/$old_name.sha256" "$local_stale/$old_name.sha256" if run_pull "$remote_directory" "$local_stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then record_failure 'stale backup unexpectedly succeeded' elif ! grep -q 'Newest encrypted backup is stale or has a future timestamp' "$test_root/stale.log"; then record_failure 'stale backup failed for an unexpected reason' fi [[ -f "$local_stale/$old_name" && -f "$local_stale/$old_name.sha256" ]] || record_failure 'failed stale pull pruned an existing local backup' bad_checksum_name="krona-staff-$fresh_stamp-103.dump.cms" make_backup "$bad_checksum_name" printf '%064d %s\n' 0 "$bad_checksum_name" > "$remote_store/$bad_checksum_name.sha256" if run_pull "$remote_directory" "$test_root/local-bad-checksum" "$bad_checksum_name" "$test_root/scp-bad.count" > "$test_root/bad-checksum.log" 2>&1; then record_failure 'checksum mismatch unexpectedly succeeded' elif ! grep -q 'Transfer checksum mismatch' "$test_root/bad-checksum.log"; then record_failure 'checksum mismatch failed for an unexpected reason' fi [[ ! -e "$test_root/local-bad-checksum/$bad_checksum_name" ]] || record_failure 'checksum mismatch left a completed backup' assert_rejected_before_ssh 'remote root' / "$test_root/local-remote-root" "$test_root/scp-root.count" assert_rejected_before_ssh 'remote traversal' /srv/krona-staff/../escape/ "$test_root/local-remote-traversal" "$test_root/scp-traversal.count" assert_rejected_before_ssh 'local root' "$remote_directory" / "$test_root/scp-local-root.count" local_traversal="$test_root/local-traversal/../local-target" assert_rejected_before_ssh 'local traversal' "$remote_directory" "$local_traversal" "$test_root/scp-local-traversal.count" local_directory_symlink="$test_root/local-directory-symlink" ln -s "$local_success" "$local_directory_symlink" assert_rejected_before_ssh 'local directory symlink' "$remote_directory" "$local_directory_symlink" "$test_root/scp-local-symlink.count" scp_failure_name="krona-staff-$fresh_stamp-104.dump.cms" make_backup "$scp_failure_name" if run_pull "$remote_directory" "$test_root/local-scp-failure" "$scp_failure_name" "$test_root/scp-failure.count" MOCK_SCP_FAIL_AT=2 > "$test_root/scp-failure.log" 2>&1; then record_failure 'scp failure unexpectedly succeeded' fi [[ ! -e "$test_root/local-scp-failure/$scp_failure_name" ]] || record_failure 'scp failure left a completed local backup' [[ ! -e "$test_root/local-scp-failure/$scp_failure_name.sha256" ]] || record_failure 'scp failure left a completed local checksum' if (( assertion_failures > 0 )); then printf 'backup pull tests had %s failure(s)\n' "$assertion_failures" >&2 exit 1 fi printf 'backup pull tests passed: CMS+checksum, idempotence, stale/mismatch rejection, path validation, scp failure\n'