214 lines
10 KiB
Bash
214 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
repo_root=$(cd -- "$(dirname -- "$0")/.." && pwd -P)
|
|
pull_script="$repo_root/scripts/pull-pilot-backups.sh"
|
|
test_parent=$(cd -- /tmp && pwd -P)
|
|
test_root=$(mktemp -d "$test_parent/krona-backup-pull-test.XXXXXX")
|
|
test_root=$(cd -- "$test_root" && pwd -P)
|
|
is_test_root() {
|
|
[ "$(dirname -- "$1")" = "$test_parent" ] || return 1
|
|
case "$(basename -- "$1")" in krona-backup-pull-test.*) return 0 ;; *) return 1 ;; esac
|
|
}
|
|
if ! is_test_root "$test_root"; then
|
|
printf 'unexpected temporary test directory: %s\n' "$test_root" >&2
|
|
exit 1
|
|
fi
|
|
mock_bin="$test_root/bin"
|
|
remote_store="$test_root/mock-remote"
|
|
mkdir -p "$mock_bin" "$remote_store"
|
|
cleanup() {
|
|
resolved_test_root=$(cd -- "$test_root" && pwd -P) || return 1
|
|
if ! is_test_root "$resolved_test_root"; then
|
|
printf 'refusing to remove unexpected test directory: %s\n' "$resolved_test_root" >&2
|
|
return 1
|
|
fi
|
|
rm -rf -- "$resolved_test_root"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
cat > "$mock_bin/ssh" <<'MOCK_SSH'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
[[ "$#" -eq 5 && "$1" == -o && "$2" == BatchMode=yes && "$3" == -- ]] || exit 61
|
|
printf '%s\n' "$4" "$5" >> "$MOCK_SSH_LOG"
|
|
printf '%s' "$MOCK_REMOTE_LISTING"
|
|
MOCK_SSH
|
|
|
|
cat > "$mock_bin/scp" <<'MOCK_SCP'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
[[ "$#" -eq 4 && "$1" == -B && "$2" == -- ]] || exit 62
|
|
count=0
|
|
[[ ! -f "$MOCK_SCP_STATE" ]] || read -r count < "$MOCK_SCP_STATE"
|
|
count=$((count + 1))
|
|
printf '%s\n' "$count" > "$MOCK_SCP_STATE"
|
|
if [[ "${MOCK_SCP_FAIL_AT:-0}" == "$count" ]]; then exit 63; fi
|
|
source_name=${3##*/}
|
|
cp -- "$MOCK_REMOTE_STORE/$source_name" "$4"
|
|
MOCK_SCP
|
|
chmod +x "$mock_bin/ssh" "$mock_bin/scp"
|
|
export PATH="$mock_bin:$PATH"
|
|
|
|
cert="$test_root/recipient.crt"
|
|
key="$test_root/recipient.key"
|
|
/usr/bin/openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 1 \
|
|
-keyout "$key" -out "$cert" -subj '/CN=pull-test-recipient' >/dev/null 2>&1
|
|
|
|
remote_directory=/srv/krona-staff/backups/
|
|
ssh_log="$test_root/ssh.log"
|
|
success_state="$test_root/scp-success.count"
|
|
assertion_failures=0
|
|
|
|
run_pull() {
|
|
local remote_dir=$1 local_dir=$2 listing=$3 state=$4
|
|
shift 4
|
|
env PATH="$mock_bin:$PATH" \
|
|
PILOT_SSH_TARGET=pull-test@invalid \
|
|
PILOT_REMOTE_BACKUP_DIRECTORY="$remote_dir" \
|
|
BACKUP_LOCAL_DIRECTORY="$local_dir" \
|
|
BACKUP_MAX_AGE_HOURS=36 \
|
|
BACKUP_LOCAL_RETENTION_DAYS=90 \
|
|
MOCK_REMOTE_STORE="$remote_store" \
|
|
MOCK_REMOTE_LISTING="$listing" \
|
|
MOCK_SCP_STATE="$state" \
|
|
MOCK_SSH_LOG="$ssh_log" \
|
|
"$@" bash "$pull_script"
|
|
}
|
|
|
|
record_failure() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
assertion_failures=$((assertion_failures + 1))
|
|
}
|
|
|
|
make_backup() {
|
|
local name=$1
|
|
printf 'synthetic PostgreSQL backup\n' | /usr/bin/openssl cms -encrypt -binary \
|
|
-aes-256-cbc -outform DER -stream -in /dev/stdin \
|
|
-out "$remote_store/$name" "$cert"
|
|
(cd -- "$remote_store" && sha256sum -- "$name" > "$name.sha256")
|
|
}
|
|
|
|
assert_rejected_before_ssh() {
|
|
local label=$1 remote_dir=$2 local_dir=$3 state=$4
|
|
shift 4
|
|
rm -f -- "$ssh_log"
|
|
if run_pull "$remote_dir" "$local_dir" "$fresh_name" "$state" "$@" > "$test_root/reject.log" 2>&1; then
|
|
record_failure "$label unexpectedly succeeded"
|
|
elif [[ -e "$ssh_log" ]]; then
|
|
record_failure "$label reached mocked ssh before rejection"
|
|
fi
|
|
}
|
|
|
|
[[ "$(command -v ssh)" == "$mock_bin/ssh" ]] || { echo 'mock ssh is not first in PATH' >&2; exit 1; }
|
|
[[ "$(command -v scp)" == "$mock_bin/scp" ]] || { echo 'mock scp is not first in PATH' >&2; exit 1; }
|
|
bash -n "$pull_script"
|
|
|
|
fresh_stamp=$(date -u '+%Y%m%dT%H%M%SZ')
|
|
fresh_name="krona-staff-$fresh_stamp-101.dump.cms"
|
|
make_backup "$fresh_name"
|
|
local_success="$test_root/local-success"
|
|
first_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$success_state")
|
|
[[ -f "$local_success/$fresh_name" && -f "$local_success/$fresh_name.sha256" ]] || {
|
|
record_failure 'fresh CMS and checksum were not installed locally'
|
|
}
|
|
cmp -s "$remote_store/$fresh_name" "$local_success/$fresh_name" || record_failure 'fresh CMS differs from remote bytes'
|
|
(cd -- "$local_success" && sha256sum -c -- "$fresh_name.sha256") >/dev/null || {
|
|
record_failure 'fresh local checksum does not validate'
|
|
}
|
|
[[ "$first_output" == *'newly_copied=1'* ]] || record_failure 'first pull did not report one new copy'
|
|
[[ "$(cat "$success_state")" == 2 ]] || record_failure 'first pull did not fetch exactly CMS and checksum'
|
|
|
|
second_output=$(run_pull "$remote_directory" "$local_success" "$fresh_name" "$success_state")
|
|
[[ "$second_output" == *'newly_copied=0'* ]] || record_failure 'repeat pull was not idempotent'
|
|
[[ "$(cat "$success_state")" == 2 ]] || record_failure 'repeat pull issued additional scp calls'
|
|
|
|
old_stamp=$(date -u -d '91 days ago' '+%Y%m%dT%H%M%SZ')
|
|
old_name="krona-staff-$old_stamp-201.dump.cms"
|
|
recent_stamp=$(date -u -d '10 days ago' '+%Y%m%dT%H%M%SZ')
|
|
recent_name="krona-staff-$recent_stamp-202.dump.cms"
|
|
unverified_name="krona-staff-$old_stamp-203.dump.cms"
|
|
symlink_name="krona-staff-$old_stamp-204.dump.cms"
|
|
make_backup "$old_name"
|
|
make_backup "$recent_name"
|
|
make_backup "$unverified_name"
|
|
make_backup "$symlink_name"
|
|
retention_local="$test_root/local-retention"
|
|
mkdir -p "$retention_local"
|
|
for name in "$old_name" "$recent_name" "$unverified_name"; do
|
|
cp -- "$remote_store/$name" "$retention_local/$name"
|
|
cp -- "$remote_store/$name.sha256" "$retention_local/$name.sha256"
|
|
done
|
|
printf 'tampered synthetic bytes\n' >> "$retention_local/$unverified_name"
|
|
printf 'preserve unrelated file\n' > "$retention_local/operator-notes.txt"
|
|
printf 'symlink target\n' > "$test_root/symlink-target.bin"
|
|
ln -s "$test_root/symlink-target.bin" "$retention_local/$symlink_name"
|
|
cp -- "$remote_store/$symlink_name.sha256" "$retention_local/$symlink_name.sha256"
|
|
retention_output=$(run_pull "$remote_directory" "$retention_local" "$fresh_name" "$test_root/scp-retention.count" BACKUP_LOCAL_RETENTION_DAYS=90)
|
|
[[ ! -e "$retention_local/$old_name" && ! -e "$retention_local/$old_name.sha256" ]] ||
|
|
record_failure 'verified backup older than 90 days was not pruned with its checksum'
|
|
[[ -f "$retention_local/$recent_name" && -f "$retention_local/$recent_name.sha256" ]] ||
|
|
record_failure 'recent backup was pruned'
|
|
[[ -f "$retention_local/$unverified_name" && -f "$retention_local/$unverified_name.sha256" ]] ||
|
|
record_failure 'unverified old backup was removed'
|
|
[[ -L "$retention_local/$symlink_name" && -f "$test_root/symlink-target.bin" ]] ||
|
|
record_failure 'symlink backup or its target was changed'
|
|
[[ -f "$retention_local/$symlink_name.sha256" && -f "$retention_local/operator-notes.txt" ]] ||
|
|
record_failure 'retention removed a symlink sidecar or unrelated file'
|
|
[[ "$retention_output" == *'retention_pruned=1'* ]] || record_failure 'retention did not report one pruned backup'
|
|
[[ "$retention_output" == *'retention_skipped=1'* ]] || record_failure 'retention did not report the unverified backup'
|
|
|
|
assert_rejected_before_ssh 'zero retention' "$remote_directory" "$test_root/local-invalid-retention" "$test_root/scp-invalid-retention.count" BACKUP_LOCAL_RETENTION_DAYS=0
|
|
grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" ||
|
|
record_failure 'invalid retention failed for an unexpected reason'
|
|
assert_rejected_before_ssh 'excessive retention' "$remote_directory" "$test_root/local-excessive-retention" "$test_root/scp-excessive-retention.count" BACKUP_LOCAL_RETENTION_DAYS=36501
|
|
grep -q 'BACKUP_LOCAL_RETENTION_DAYS must be an integer' "$test_root/reject.log" ||
|
|
record_failure 'excessive retention failed for an unexpected reason'
|
|
|
|
stale_stamp=$(date -u -d '48 hours ago' '+%Y%m%dT%H%M%SZ')
|
|
stale_name="krona-staff-$stale_stamp-102.dump.cms"
|
|
make_backup "$stale_name"
|
|
local_stale="$test_root/local-stale"
|
|
mkdir -p "$local_stale"
|
|
cp -- "$remote_store/$old_name" "$local_stale/$old_name"
|
|
cp -- "$remote_store/$old_name.sha256" "$local_stale/$old_name.sha256"
|
|
if run_pull "$remote_directory" "$local_stale" "$stale_name" "$test_root/scp-stale.count" > "$test_root/stale.log" 2>&1; then
|
|
record_failure 'stale backup unexpectedly succeeded'
|
|
elif ! grep -q 'Newest encrypted backup is stale or has a future timestamp' "$test_root/stale.log"; then
|
|
record_failure 'stale backup failed for an unexpected reason'
|
|
fi
|
|
[[ -f "$local_stale/$old_name" && -f "$local_stale/$old_name.sha256" ]] ||
|
|
record_failure 'failed stale pull pruned an existing local backup'
|
|
|
|
bad_checksum_name="krona-staff-$fresh_stamp-103.dump.cms"
|
|
make_backup "$bad_checksum_name"
|
|
printf '%064d %s\n' 0 "$bad_checksum_name" > "$remote_store/$bad_checksum_name.sha256"
|
|
if run_pull "$remote_directory" "$test_root/local-bad-checksum" "$bad_checksum_name" "$test_root/scp-bad.count" > "$test_root/bad-checksum.log" 2>&1; then
|
|
record_failure 'checksum mismatch unexpectedly succeeded'
|
|
elif ! grep -q 'Transfer checksum mismatch' "$test_root/bad-checksum.log"; then
|
|
record_failure 'checksum mismatch failed for an unexpected reason'
|
|
fi
|
|
[[ ! -e "$test_root/local-bad-checksum/$bad_checksum_name" ]] || record_failure 'checksum mismatch left a completed backup'
|
|
|
|
assert_rejected_before_ssh 'remote root' / "$test_root/local-remote-root" "$test_root/scp-root.count"
|
|
assert_rejected_before_ssh 'remote traversal' /srv/krona-staff/../escape/ "$test_root/local-remote-traversal" "$test_root/scp-traversal.count"
|
|
assert_rejected_before_ssh 'local root' "$remote_directory" / "$test_root/scp-local-root.count"
|
|
local_traversal="$test_root/local-traversal/../local-target"
|
|
assert_rejected_before_ssh 'local traversal' "$remote_directory" "$local_traversal" "$test_root/scp-local-traversal.count"
|
|
local_directory_symlink="$test_root/local-directory-symlink"
|
|
ln -s "$local_success" "$local_directory_symlink"
|
|
assert_rejected_before_ssh 'local directory symlink' "$remote_directory" "$local_directory_symlink" "$test_root/scp-local-symlink.count"
|
|
|
|
scp_failure_name="krona-staff-$fresh_stamp-104.dump.cms"
|
|
make_backup "$scp_failure_name"
|
|
if run_pull "$remote_directory" "$test_root/local-scp-failure" "$scp_failure_name" "$test_root/scp-failure.count" MOCK_SCP_FAIL_AT=2 > "$test_root/scp-failure.log" 2>&1; then
|
|
record_failure 'scp failure unexpectedly succeeded'
|
|
fi
|
|
[[ ! -e "$test_root/local-scp-failure/$scp_failure_name" ]] || record_failure 'scp failure left a completed local backup'
|
|
[[ ! -e "$test_root/local-scp-failure/$scp_failure_name.sha256" ]] || record_failure 'scp failure left a completed local checksum'
|
|
|
|
if (( assertion_failures > 0 )); then
|
|
printf 'backup pull tests had %s failure(s)\n' "$assertion_failures" >&2
|
|
exit 1
|
|
fi
|
|
printf 'backup pull tests passed: CMS+checksum, idempotence, stale/mismatch rejection, path validation, scp failure\n'
|