b897b0ea88
The update guard compared the archive version with PROGRAM_VERSION using shell
string ordering. Both halves were wrong.
PROGRAM_VERSION is a constant describing the bootstrap source tree ("0.2.0-dev"),
never the deployed component, so it could not represent what is installed. The
cross-role compatibility check compared it with a manifest field such as
"v0.1.0-alpha.9"; those can never be equal, so update-control on a host that also
runs the agent always failed. The downgrade guard compared the same mismatched
pair and passed only by accident, because "v" sorts above "0" in ASCII.
String ordering is also wrong for the version scheme in use: "0.1.0-alpha.10"
sorts below "0.1.0-alpha.9", so the next release after the ninth would have been
rejected as a downgrade.
- record the installed version per role at install and update time, and compare
against that instead of PROGRAM_VERSION;
- order versions with sort -V after stripping the leading "v", so prerelease
numbering and tag prefixes compare correctly;
- treat an unknown peer version as a warning rather than a failure, because
installations predating version recording have nothing to compare against;
- guard all of the above in the bootstrap contract test, including the six
ordering cases and a check that the lexicographic comparison is not restored.
Verified by deliberately reintroducing each defect: lexicographic comparison,
sort without -V, and a missing version record are all caught by the contract test.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
872 lines
42 KiB
Bash
Executable file
872 lines
42 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
IFS=$'\n\t'
|
|
|
|
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
|
|
helper="$root/deploy/ochenstarik-smm-policy-apply"
|
|
emergency="$root/deploy/ochenstarik-smm-emergency"
|
|
acceptance="$root/tests/acceptance/three-server-mesh.sh"
|
|
|
|
grep -Fq 'listing="$(/usr/sbin/nft -a list chain' "$helper" || {
|
|
printf '%s\n' "policy status probe must fail closed when nftables cannot be inspected" >&2
|
|
exit 1
|
|
}
|
|
grep -Fq "grep -Eiq 'No such file or directory|does not exist'" "$helper"
|
|
provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service"
|
|
|
|
grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit"
|
|
grep -Fq 'ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback' "$provisioning_helper_unit"
|
|
grep -Fq 'install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback"' "$bootstrap"
|
|
if grep -Fq 'SMM_EnrollToken=$ENROLL_TOKEN' "$bootstrap"; then
|
|
printf '%s\n' "enrollment token is exposed through process argv" >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'readonly ENROLLMENT_DIR="${STATE_DIR}-enrollment"' "$bootstrap"
|
|
grep -Fq 'install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$ENROLLMENT_DIR"' "$bootstrap"
|
|
grep -Fq 'token_temp="$(mktemp "$ENROLLMENT_DIR/.enroll-token.XXXXXXXX")"' "$bootstrap"
|
|
if grep -Fq '$STATE_DIR/enrollment' "$bootstrap"; then
|
|
printf '%s\n' "enrollment directory is beneath Control-writable state" >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'chmod 0400 "$token_temp"' "$bootstrap"
|
|
grep -Fq 'mv -fT -- "$token_temp" "$token_file"' "$bootstrap"
|
|
grep -Fq '"SMM_EnrollTokenFile=$token_file"' "$bootstrap"
|
|
grep -Fq 'rm -f -- "$token_file"' "$bootstrap"
|
|
grep -Fq 'rm -f -- "$ENROLLMENT_TOKEN_FILE"' "$bootstrap"
|
|
grep -Fq 'rm -f -- "$ENROLLMENT_TOKEN_TEMP"' "$bootstrap"
|
|
grep -Fq 'SMM_AgentUid=$(id -u "$AGENT_USER")' "$bootstrap"
|
|
grep -Fq 'refresh_agent_uid() {' "$bootstrap"
|
|
grep -Fq 'agent_uid="$(id -u "$AGENT_USER")"' "$bootstrap"
|
|
grep -Fq "printf 'SMM_AgentUid=%s\\n' \"\$agent_uid\"" "$bootstrap"
|
|
grep -Fq ' refresh_agent_uid' "$bootstrap"
|
|
grep -Fq 'temp="$(mktemp "$ETC_DIR/.agent.env.XXXXXXXX")"' "$bootstrap"
|
|
grep -Fq 'mv -fT -- "$temp" "$env_file"' "$bootstrap"
|
|
refresh_line="$(grep -F -m1 -n ' refresh_agent_uid' "$bootstrap" | cut -d: -f1)"
|
|
stop_line="$(grep -F -m1 -n ' systemctl stop "$unit"' "$bootstrap" | cut -d: -f1)"
|
|
(( refresh_line < stop_line ))
|
|
|
|
help_output="$(bash "$bootstrap" --help)"
|
|
version_output="$(bash "$bootstrap" --version)"
|
|
|
|
grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output"
|
|
grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output"
|
|
grep -Fq "install-node ARCHIVE" <<<"$help_output"
|
|
grep -Fq "mesh-init PUBLIC_ENDPOINT" <<<"$help_output"
|
|
grep -Fq "peer-add SMMPEER1_CODE" <<<"$help_output"
|
|
grep -Fq "mesh-status" <<<"$help_output"
|
|
grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output"
|
|
grep -Fq "node-code NODE_ID" <<<"$help_output"
|
|
grep -Fq "control-device-code DEVICE_ID" <<<"$help_output"
|
|
grep -Fq "verify-release ARCHIVE" <<<"$help_output"
|
|
grep -Fq "node-token NODE_ID" <<<"$help_output"
|
|
grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output"
|
|
|
|
extract_bootstrap_function() {
|
|
local name="$1"
|
|
awk -v signature="$name() {" '
|
|
$0 == signature { emitting = 1 }
|
|
emitting { print }
|
|
emitting && $0 == "}" { exit }
|
|
' "$bootstrap"
|
|
}
|
|
validate_port_definition="$(extract_bootstrap_function validate_port)"
|
|
validate_ipv4_literal_definition="$(extract_bootstrap_function validate_ipv4_literal)"
|
|
validate_control_url_definition="$(extract_bootstrap_function validate_control_url)"
|
|
for accepted_url in \
|
|
https://example.com \
|
|
https://host.example:7443 \
|
|
https://10.0.0.1:7443 \
|
|
'https://[2001:db8::1]:7443' \
|
|
'https://[::1]:7443' \
|
|
'https://[2001:db8::]' \
|
|
'https://[::]' \
|
|
'https://[::ffff:192.0.2.128]' \
|
|
'https://[2001:db8:3:4::192.0.2.33]:7443' \
|
|
'https://[1:2:3:4:5:6:192.0.2.1]' \
|
|
https://example.com/; do
|
|
if ! (fail() { exit 1; }; source <(printf '%s\n%s\n%s\n' "$validate_port_definition" "$validate_ipv4_literal_definition" "$validate_control_url_definition"); validate_control_url "$accepted_url"); then
|
|
printf 'valid Control URL was rejected: %s\n' "$accepted_url" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
for rejected_url in \
|
|
http://example.com \
|
|
https:// \
|
|
https://:7443 \
|
|
'https://[2001:db8::1' \
|
|
'https://[::::]' \
|
|
'https://[1:2:3]' \
|
|
'https://[::ffff:192.0.2.999]' \
|
|
'https://[::ffff:192.0.2]' \
|
|
'https://[::ffff:192.0.2.1.5]' \
|
|
'https://[::ffff:192.0.2.x]' \
|
|
'https://[1:2:3:4:5:6:7:192.0.2.1]' \
|
|
'https://[::ffff:192.0.2.1:]' \
|
|
'https://[1:2:3:4:5:6:192.0.2.1:]' \
|
|
'https://[1:2:3:4:5:6:7:8:]' \
|
|
'https://[:1:2:3:4:5:6:7]' \
|
|
'https://[::ffff:18446744073709551617.0.0.1]' \
|
|
'https://[::1]:18446744073709551696' \
|
|
'https://2001:db8::1:7443' \
|
|
'https://example.com:7443:7444' \
|
|
https://example..com \
|
|
https://999.0.0.1 \
|
|
https://18446744073709551617.0.0.1 \
|
|
https://example.com/path \
|
|
https://user@example.com \
|
|
'https://example.com:0' \
|
|
'https://example.com:65536' \
|
|
'https://example.com:18446744073709551696' \
|
|
'https://example.com?query=1' \
|
|
'https://example.com#fragment'; do
|
|
if (fail() { exit 1; }; source <(printf '%s\n%s\n%s\n' "$validate_port_definition" "$validate_ipv4_literal_definition" "$validate_control_url_definition"); validate_control_url "$rejected_url"); then
|
|
printf 'invalid Control URL was accepted: %s\n' "$rejected_url" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
for action in status version --version preflight help -h --help; do
|
|
if bash "$bootstrap" "$action" surplus >/dev/null 2>&1; then
|
|
printf 'bootstrap action accepted surplus arguments: %s\n' "$action" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if ! bash "$bootstrap" >/dev/null 2>&1; then
|
|
printf '%s\n' 'bootstrap no-argument help form failed' >&2
|
|
exit 1
|
|
fi
|
|
for action in verify-release install-control install-agent install-node mesh-init peer-add \
|
|
update-control update-agent rollback node-code control-device-code node-token uninstall-control; do
|
|
if bash "$bootstrap" "$action" >/dev/null 2>&1; then
|
|
printf 'bootstrap action accepted missing arguments: %s\n' "$action" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
base64url_encode_definition="$(extract_bootstrap_function base64url_encode)"
|
|
create_device_code_definition="$(extract_bootstrap_function create_device_code)"
|
|
device_fixture="$(mktemp -d -t smm-device-code.XXXXXXXX)"
|
|
printf '%s\n' 'https://control.example:7443' >"$device_fixture/control-public-url"
|
|
printf '%s\n' 'fixture-ca' >"$device_fixture/control-ca.crt"
|
|
device_code="$(
|
|
ETC_DIR="$device_fixture"
|
|
require_root() { :; }
|
|
require_command() { :; }
|
|
validate_node_id() { [[ "$1" == 'desktop-device' ]]; }
|
|
validate_control_url() { [[ "$1" == 'https://control.example:7443' ]]; }
|
|
run_control_cli() {
|
|
[[ "$1" == 'device-token-create' && "$2" == 'desktop-device' ]]
|
|
printf '%043d\n' 0
|
|
}
|
|
openssl() { printf '%s' 'DER-fixture'; }
|
|
source <(printf '%s\n%s\n' "$base64url_encode_definition" "$create_device_code_definition")
|
|
create_device_code desktop-device
|
|
)"
|
|
[[ "$device_code" == SMMDEV1-* ]]
|
|
device_payload_encoded="${device_code#SMMDEV1-}"
|
|
device_payload_encoded="${device_payload_encoded//-/+}"
|
|
device_payload_encoded="${device_payload_encoded//_/\/}"
|
|
case $(( ${#device_payload_encoded} % 4 )) in
|
|
0) ;;
|
|
2) device_payload_encoded+='==' ;;
|
|
3) device_payload_encoded+='=' ;;
|
|
*) printf '%s\n' 'invalid generated SMMDEV1 base64url length' >&2; exit 1 ;;
|
|
esac
|
|
device_payload="$(printf '%s' "$device_payload_encoded" | base64 -d)"
|
|
grep -Fxq 'VERSION=1' <<<"$device_payload"
|
|
grep -Fxq 'DEVICE=desktop-device' <<<"$device_payload"
|
|
grep -Fxq 'TOKEN=0000000000000000000000000000000000000000000' <<<"$device_payload"
|
|
grep -Fxq 'URL=https://control.example:7443' <<<"$device_payload"
|
|
grep -Fxq "CA=$(printf '%s' 'DER-fixture' | base64 -w 0)" <<<"$device_payload"
|
|
rm -rf -- "$device_fixture"
|
|
|
|
grep -Fq 'readonly BOOTSTRAP_COMMAND="/usr/local/sbin/ochenstarik-server-monitor-manager.sh"' "$bootstrap"
|
|
grep -Fq 'staging="$(mktemp "$(dirname "$BOOTSTRAP_COMMAND")/.ochenstarik-server-monitor-manager.XXXXXXXX")"' "$bootstrap"
|
|
grep -Fq 'mv -fT -- "$staging" "$BOOTSTRAP_COMMAND"' "$bootstrap"
|
|
[[ "$(grep -Fc ' install_bootstrap_command' "$bootstrap")" -eq 3 ]]
|
|
update_role_definition="$(extract_bootstrap_function update_role)"
|
|
validation_line="$(grep -n -m1 'validate_control_state_migration' <<<"$update_role_definition" | cut -d: -f1)"
|
|
environment_validation_line="$(grep -n -m1 'validate_control_environment_migration' <<<"$update_role_definition" | cut -d: -f1)"
|
|
control_stop_line="$(grep -n -m1 'systemctl stop \"\$unit\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
(( validation_line < control_stop_line && environment_validation_line < control_stop_line ))
|
|
guard_arm_line="$(grep -n -m1 'CONTROL_UPDATE_RECOVERY_REQUIRED=1' <<<"$update_role_definition" | cut -d: -f1)"
|
|
guard_clear_line="$(grep -n -m1 'CONTROL_UPDATE_RECOVERY_REQUIRED=0' <<<"$update_role_definition" | cut -d: -f1)"
|
|
prepare_line="$(grep -n -m1 'prepare_control_state' <<<"$update_role_definition" | cut -d: -f1)"
|
|
rewrite_line="$(grep -n -m1 'rewrite_control_environment' <<<"$update_role_definition" | cut -d: -f1)"
|
|
bootstrap_line="$(grep -n -m1 'install_bootstrap_command' <<<"$update_role_definition" | cut -d: -f1)"
|
|
unit_line="$(grep -n -m1 'install_unit \"\$TEMP_DIR/deploy/\$CONTROL_UNIT\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
enable_line="$(grep -n -m1 'systemctl enable \"\$CONTROL_UNIT\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
binary_line="$(grep -n -m1 'install_tree_atomic \"\$TEMP_DIR/control\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
restart_line="$(grep -n -m1 'systemctl restart \"\$unit\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
active_line="$(grep -n -m1 'systemctl is-active --quiet \"\$unit\"' <<<"$update_role_definition" | cut -d: -f1)"
|
|
for guarded_line in "$control_stop_line" "$prepare_line" "$rewrite_line" "$bootstrap_line" \
|
|
"$unit_line" "$enable_line" "$binary_line" "$restart_line" "$active_line"; do
|
|
(( guard_arm_line < guarded_line && guarded_line < guard_clear_line ))
|
|
done
|
|
grep -Fq 'install_bootstrap_command' <<<"$update_role_definition"
|
|
grep -Fq 'install_unit "$TEMP_DIR/deploy/$CONTROL_UNIT" "$CONTROL_UNIT"' <<<"$update_role_definition"
|
|
grep -Fq 'install_tree_atomic "$TEMP_DIR/control" "$LIB_DIR/control" "$user"' <<<"$update_role_definition"
|
|
grep -Fq 'systemctl restart "$unit"' <<<"$update_role_definition"
|
|
grep -Fq 'systemctl is-active --quiet "$unit"' <<<"$update_role_definition"
|
|
grep -Fq 'install -d -m 0711 -o root -g root "$ETC_DIR" "$STATE_DIR"' "$bootstrap"
|
|
grep -Fq 'install -d -m 0700 -o "$CONTROL_USER" -g "$CONTROL_USER" "$STATE_DIR/control"' "$bootstrap"
|
|
grep -Fq 'Control__DatabasePath=$STATE_DIR/control/control.db' "$bootstrap"
|
|
grep -Fq 'Control__BackupDirectory=$STATE_DIR/control/backups' "$bootstrap"
|
|
validate_control_state_migration_definition="$(extract_bootstrap_function validate_control_state_migration)"
|
|
prepare_control_state_definition="$(extract_bootstrap_function prepare_control_state)"
|
|
control_state_fixture="$(mktemp -d -t smm-control-state.XXXXXXXX)"
|
|
mkdir -p "$control_state_fixture/backups"
|
|
printf '%s' database >"$control_state_fixture/control.db"
|
|
printf '%s' wal >"$control_state_fixture/control.db-wal"
|
|
printf '%s' backup >"$control_state_fixture/backups/manifest.json"
|
|
(
|
|
STATE_DIR="$control_state_fixture"
|
|
CONTROL_USER=fixture
|
|
fail() { printf '%s\n' "$*" >&2; exit 1; }
|
|
install() {
|
|
local arguments=()
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
-m|-o|-g) shift 2 ;;
|
|
*) arguments+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
command install "${arguments[@]}"
|
|
}
|
|
chown() { :; }
|
|
source <(printf '%s\n%s\n' "$validate_control_state_migration_definition" "$prepare_control_state_definition")
|
|
validate_control_state_migration
|
|
prepare_control_state
|
|
)
|
|
[[ ! -e "$control_state_fixture/control.db" ]]
|
|
[[ "$(<"$control_state_fixture/control/control.db")" == database ]]
|
|
[[ "$(<"$control_state_fixture/control/control.db-wal")" == wal ]]
|
|
[[ "$(<"$control_state_fixture/control/backups/manifest.json")" == backup ]]
|
|
if [[ "$(uname -s)" != MINGW* ]]; then
|
|
[[ "$(stat -c '%a' "$control_state_fixture/control")" == 700 ]]
|
|
[[ "$(stat -c '%a' "$control_state_fixture/control/control.db")" == 600 ]]
|
|
fi
|
|
printf '%s' conflict >"$control_state_fixture/control.db"
|
|
if (
|
|
STATE_DIR="$control_state_fixture"
|
|
fail() { exit 1; }
|
|
source <(printf '%s\n' "$validate_control_state_migration_definition")
|
|
validate_control_state_migration
|
|
); then
|
|
printf '%s\n' 'conflicting legacy and role-isolated Control state was accepted' >&2
|
|
exit 1
|
|
fi
|
|
rm -rf -- "$control_state_fixture"
|
|
|
|
validate_control_environment_migration_definition="$(extract_bootstrap_function validate_control_environment_migration)"
|
|
rewrite_control_environment_definition="$(extract_bootstrap_function rewrite_control_environment)"
|
|
alpha7_fixture="$(mktemp -d -t smm-alpha7-update.XXXXXXXX)"
|
|
mkdir -p "$alpha7_fixture/state/backups" "$alpha7_fixture/etc"
|
|
printf '%s' alpha7-db >"$alpha7_fixture/state/control.db"
|
|
printf '%s' alpha7-wal >"$alpha7_fixture/state/control.db-wal"
|
|
printf '%s' alpha7-shm >"$alpha7_fixture/state/control.db-shm"
|
|
printf '%s' alpha7-backup >"$alpha7_fixture/state/backups/manifest.json"
|
|
cat >"$alpha7_fixture/etc/control.env" <<EOF
|
|
# alpha.7 fixture: preserve comments and every unrelated value
|
|
ASPNETCORE_URLS=https://0.0.0.0:7443
|
|
Control__DatabasePath=$alpha7_fixture/state/control.db
|
|
Control__CertificateAuthorityPath=/custom/control-ca.pfx
|
|
Control__BackupDirectory=$alpha7_fixture/state/backups
|
|
Control__LinkReconciliationSeconds=777
|
|
CUSTOM_VALUE=spaces are preserved exactly
|
|
EOF
|
|
cp "$alpha7_fixture/etc/control.env" "$alpha7_fixture/original.env"
|
|
(
|
|
STATE_DIR="$alpha7_fixture/state"
|
|
ETC_DIR="$alpha7_fixture/etc"
|
|
CONTROL_USER=fixture
|
|
fail() { printf '%s\n' "$*" >&2; exit 1; }
|
|
install() {
|
|
local arguments=()
|
|
while (( $# > 0 )); do
|
|
case "$1" in -m|-o|-g) shift 2 ;; *) arguments+=("$1"); shift ;; esac
|
|
done
|
|
command install "${arguments[@]}"
|
|
}
|
|
chown() { :; }
|
|
source <(printf '%s\n%s\n%s\n%s\n' \
|
|
"$validate_control_state_migration_definition" \
|
|
"$prepare_control_state_definition" \
|
|
"$validate_control_environment_migration_definition" \
|
|
"$rewrite_control_environment_definition")
|
|
validate_control_state_migration
|
|
validate_control_environment_migration
|
|
prepare_control_state
|
|
rewrite_control_environment
|
|
)
|
|
[[ "$(<"$alpha7_fixture/state/control/control.db")" == alpha7-db ]]
|
|
[[ "$(<"$alpha7_fixture/state/control/control.db-wal")" == alpha7-wal ]]
|
|
[[ "$(<"$alpha7_fixture/state/control/control.db-shm")" == alpha7-shm ]]
|
|
[[ "$(<"$alpha7_fixture/state/control/backups/manifest.json")" == alpha7-backup ]]
|
|
expected_env="$(sed \
|
|
-e "s|^Control__DatabasePath=.*|Control__DatabasePath=$alpha7_fixture/state/control/control.db|" \
|
|
-e "s|^Control__BackupDirectory=.*|Control__BackupDirectory=$alpha7_fixture/state/control/backups|" \
|
|
"$alpha7_fixture/original.env")"
|
|
[[ "$(<"$alpha7_fixture/etc/control.env")" == "$expected_env" ]]
|
|
printf '%s\n' "Control__DatabasePath=$alpha7_fixture/state/control/control.db" \
|
|
>>"$alpha7_fixture/etc/control.env"
|
|
if (
|
|
ETC_DIR="$alpha7_fixture/etc"
|
|
fail() { exit 1; }
|
|
source <(printf '%s\n' "$validate_control_environment_migration_definition")
|
|
validate_control_environment_migration
|
|
); then
|
|
printf '%s\n' 'conflicting Control environment paths were accepted' >&2
|
|
exit 1
|
|
fi
|
|
rm -rf -- "$alpha7_fixture"
|
|
|
|
record_control_legacy_state_definition="$(extract_bootstrap_function record_control_legacy_state)"
|
|
reverse_control_state_migration_definition="$(extract_bootstrap_function reverse_control_state_migration)"
|
|
restore_control_update_backup_definition="$(extract_bootstrap_function restore_control_update_backup)"
|
|
restore_control_binary_from_archive_definition="$(extract_bootstrap_function restore_control_binary_from_archive)"
|
|
recover_control_update_definition="$(extract_bootstrap_function recover_control_update)"
|
|
[[ -n "$record_control_legacy_state_definition" ]]
|
|
[[ -n "$reverse_control_state_migration_definition" ]]
|
|
[[ -n "$restore_control_update_backup_definition" ]]
|
|
[[ -n "$restore_control_binary_from_archive_definition" ]]
|
|
[[ -n "$recover_control_update_definition" ]]
|
|
|
|
recovery_fixture="$(mktemp -d -t smm-control-recovery.XXXXXXXX)"
|
|
recovery_root="$recovery_fixture/root"
|
|
archive_root="$recovery_fixture/archive-root"
|
|
mkdir -p \
|
|
"$recovery_root/var/lib/ochenstarik-server-monitor-manager/backups" \
|
|
"$recovery_root/etc/ochenstarik-server-monitor-manager" \
|
|
"$recovery_root/etc/systemd/system" \
|
|
"$recovery_root/usr/local/lib/ochenstarik-server-monitor-manager/control" \
|
|
"$archive_root/etc/ochenstarik-server-monitor-manager" \
|
|
"$archive_root/etc/systemd/system" \
|
|
"$archive_root/usr/local/lib/ochenstarik-server-monitor-manager/control"
|
|
printf '%s' original-db >"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db"
|
|
printf '%s' original-wal >"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-wal"
|
|
printf '%s' original-shm >"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-shm"
|
|
printf '%s' original-backup >"$recovery_root/var/lib/ochenstarik-server-monitor-manager/backups/identity"
|
|
printf '%s' old-binary >"$archive_root/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control"
|
|
printf '%s\n' 'Control__DatabasePath=/var/lib/ochenstarik-server-monitor-manager/control.db' \
|
|
>"$archive_root/etc/ochenstarik-server-monitor-manager/control.env"
|
|
printf '%s' old-unit >"$archive_root/etc/systemd/system/ochenstarik-smm-control.service"
|
|
mkdir -p "$recovery_fixture/bootstrap-backups"
|
|
tar -C "$archive_root" -czf "$recovery_fixture/bootstrap-backups/alpha7.tar.gz" \
|
|
usr/local/lib/ochenstarik-server-monitor-manager/control \
|
|
etc/ochenstarik-server-monitor-manager/control.env \
|
|
etc/systemd/system/ochenstarik-smm-control.service
|
|
|
|
(
|
|
STATE_DIR="$recovery_root/var/lib/ochenstarik-server-monitor-manager"
|
|
CONTROL_USER=fixture
|
|
CONTROL_UPDATE_LEGACY_ITEMS=()
|
|
fail() { printf '%s\n' "$*" >&2; exit 1; }
|
|
install() {
|
|
local arguments=()
|
|
while (( $# > 0 )); do
|
|
case "$1" in -m|-o|-g) shift 2 ;; *) arguments+=("$1"); shift ;; esac
|
|
done
|
|
command install "${arguments[@]}"
|
|
}
|
|
chown() { :; }
|
|
source <(printf '%s\n%s\n%s\n' \
|
|
"$record_control_legacy_state_definition" \
|
|
"$prepare_control_state_definition" \
|
|
"$reverse_control_state_migration_definition")
|
|
record_control_legacy_state
|
|
prepare_control_state
|
|
[[ "$(<"$STATE_DIR/control/control.db")" == original-db ]]
|
|
reverse_control_state_migration
|
|
[[ "$(<"$STATE_DIR/control.db")" == original-db ]]
|
|
[[ "$(<"$STATE_DIR/control.db-wal")" == original-wal ]]
|
|
[[ "$(<"$STATE_DIR/control.db-shm")" == original-shm ]]
|
|
[[ "$(<"$STATE_DIR/backups/identity")" == original-backup ]]
|
|
[[ ! -e "$STATE_DIR/control/control.db" ]]
|
|
)
|
|
|
|
printf '%s' new-binary >"$recovery_root/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control"
|
|
printf '%s\n' 'Control__DatabasePath=/var/lib/ochenstarik-server-monitor-manager/control/control.db' \
|
|
>"$recovery_root/etc/ochenstarik-server-monitor-manager/control.env"
|
|
printf '%s' new-unit >"$recovery_root/etc/systemd/system/ochenstarik-smm-control.service"
|
|
(
|
|
STATE_DIR="$recovery_root/var/lib/ochenstarik-server-monitor-manager"
|
|
BACKUP_DIR="$recovery_fixture/bootstrap-backups"
|
|
CONTROL_UNIT=ochenstarik-smm-control.service
|
|
CONTROL_UPDATE_LEGACY_ITEMS=()
|
|
systemctl() { :; }
|
|
fail() { printf '%s\n' "$*" >&2; exit 1; }
|
|
source <(printf '%s\n%s\n%s\n%s\n' \
|
|
"$reverse_control_state_migration_definition" \
|
|
"$restore_control_update_backup_definition" \
|
|
"$restore_control_binary_from_archive_definition" \
|
|
"$recover_control_update_definition")
|
|
restore_control_binary_from_archive "$BACKUP_DIR/alpha7.tar.gz" "$recovery_root"
|
|
[[ "$(<"$recovery_root/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control")" == old-binary ]]
|
|
[[ "$(<"$recovery_root/etc/ochenstarik-server-monitor-manager/control.env")" == \
|
|
'Control__DatabasePath=/var/lib/ochenstarik-server-monitor-manager/control/control.db' ]]
|
|
)
|
|
|
|
rm -rf -- "$recovery_root/var/lib/ochenstarik-server-monitor-manager/control"
|
|
mkdir -p "$recovery_root/var/lib/ochenstarik-server-monitor-manager/control/backups"
|
|
mv "$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db" \
|
|
"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control/control.db"
|
|
mv "$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-wal" \
|
|
"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control/control.db-wal"
|
|
mv "$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-shm" \
|
|
"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control/control.db-shm"
|
|
mv "$recovery_root/var/lib/ochenstarik-server-monitor-manager/backups/identity" \
|
|
"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control/backups/identity"
|
|
rmdir "$recovery_root/var/lib/ochenstarik-server-monitor-manager/backups"
|
|
printf '%s' new-binary >"$recovery_root/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control"
|
|
printf '%s\n' 'Control__DatabasePath=/var/lib/ochenstarik-server-monitor-manager/control/control.db' \
|
|
>"$recovery_root/etc/ochenstarik-server-monitor-manager/control.env"
|
|
printf '%s' new-unit >"$recovery_root/etc/systemd/system/ochenstarik-smm-control.service"
|
|
(
|
|
STATE_DIR="$recovery_root/var/lib/ochenstarik-server-monitor-manager"
|
|
BACKUP_DIR="$recovery_fixture/bootstrap-backups"
|
|
CONTROL_UNIT=ochenstarik-smm-control.service
|
|
CONTROL_UPDATE_LEGACY_ITEMS=(control.db control.db-wal control.db-shm backups)
|
|
systemctl() { :; }
|
|
fail() { printf '%s\n' "$*" >&2; exit 1; }
|
|
source <(printf '%s\n%s\n%s\n' \
|
|
"$reverse_control_state_migration_definition" \
|
|
"$restore_control_update_backup_definition" \
|
|
"$recover_control_update_definition")
|
|
recover_control_update alpha7 "$recovery_root"
|
|
)
|
|
[[ "$(<"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db")" == original-db ]]
|
|
[[ "$(<"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-wal")" == original-wal ]]
|
|
[[ "$(<"$recovery_root/var/lib/ochenstarik-server-monitor-manager/control.db-shm")" == original-shm ]]
|
|
[[ "$(<"$recovery_root/var/lib/ochenstarik-server-monitor-manager/backups/identity")" == original-backup ]]
|
|
[[ "$(<"$recovery_root/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control")" == old-binary ]]
|
|
[[ "$(<"$recovery_root/etc/ochenstarik-server-monitor-manager/control.env")" == \
|
|
'Control__DatabasePath=/var/lib/ochenstarik-server-monitor-manager/control.db' ]]
|
|
[[ "$(<"$recovery_root/etc/systemd/system/ochenstarik-smm-control.service")" == old-unit ]]
|
|
rm -rf -- "$recovery_fixture"
|
|
|
|
cleanup_definition="$(extract_bootstrap_function cleanup)"
|
|
[[ -n "$cleanup_definition" ]]
|
|
for failure_step in stop state environment bootstrap unit enable binary restart active; do
|
|
guard_fixture="$(mktemp -d -t smm-control-guard.XXXXXXXX)"
|
|
marker="$guard_fixture.recovered"
|
|
set +e
|
|
(
|
|
set -Eeuo pipefail
|
|
PROGRAM=test-bootstrap
|
|
TEMP_DIR="$guard_fixture"
|
|
ENROLLMENT_TOKEN_FILE=""
|
|
ENROLLMENT_TOKEN_TEMP=""
|
|
CONTROL_UPDATE_BACKUP_ID=fixture-backup
|
|
CONTROL_UPDATE_RECOVERY_REQUIRED=1
|
|
marker="$marker"
|
|
failure_step="$failure_step"
|
|
log() { :; }
|
|
recover_control_update() { printf '%s' "$failure_step" >"$marker"; }
|
|
source <(printf '%s\n' "$cleanup_definition")
|
|
trap cleanup EXIT
|
|
false
|
|
) >/dev/null 2>&1
|
|
guard_status=$?
|
|
set -e
|
|
(( guard_status != 0 )) || {
|
|
printf 'injected Control update failure was ignored: %s\n' "$failure_step" >&2
|
|
exit 1
|
|
}
|
|
[[ -f "$marker" && "$(<"$marker")" == "$failure_step" ]] || {
|
|
printf 'Control recovery guard was not invoked for: %s\n' "$failure_step" >&2
|
|
exit 1
|
|
}
|
|
rm -f -- "$marker"
|
|
done
|
|
|
|
role_is_installed_definition="$(extract_bootstrap_function role_is_installed)"
|
|
remove_shared_ca_if_unused_definition="$(extract_bootstrap_function remove_shared_ca_if_unused)"
|
|
uninstall_agent_definition="$(extract_bootstrap_function uninstall_agent)"
|
|
uninstall_control_definition="$(extract_bootstrap_function uninstall_control)"
|
|
grep -Fq 'remove_shared_ca_if_unused' <<<"$uninstall_agent_definition"
|
|
grep -Fq 'remove_shared_ca_if_unused' <<<"$uninstall_control_definition"
|
|
if grep -Fq 'control-ca.crt' <<<"$uninstall_agent_definition$uninstall_control_definition"; then
|
|
printf '%s\n' 'a role uninstaller deletes the shared CA directly' >&2
|
|
exit 1
|
|
fi
|
|
role_fixture="$(mktemp -d -t smm-role-uninstall.XXXXXXXX)"
|
|
mkdir -p "$role_fixture/etc" "$role_fixture/lib/control" "$role_fixture/lib/agent"
|
|
printf '%s' ca >"$role_fixture/etc/control-ca.crt"
|
|
(
|
|
ETC_DIR="$role_fixture/etc"; LIB_DIR="$role_fixture/lib"
|
|
source <(printf '%s\n%s\n' "$role_is_installed_definition" "$remove_shared_ca_if_unused_definition")
|
|
rm -rf "$LIB_DIR/agent"
|
|
remove_shared_ca_if_unused
|
|
)
|
|
[[ -f "$role_fixture/etc/control-ca.crt" ]]
|
|
rm -rf "$role_fixture/lib/control"
|
|
(
|
|
ETC_DIR="$role_fixture/etc"; LIB_DIR="$role_fixture/lib"
|
|
source <(printf '%s\n%s\n' "$role_is_installed_definition" "$remove_shared_ca_if_unused_definition")
|
|
remove_shared_ca_if_unused
|
|
)
|
|
[[ ! -e "$role_fixture/etc/control-ca.crt" ]]
|
|
mkdir -p "$role_fixture/lib/control" "$role_fixture/lib/agent"
|
|
printf '%s' ca >"$role_fixture/etc/control-ca.crt"
|
|
(
|
|
ETC_DIR="$role_fixture/etc"; LIB_DIR="$role_fixture/lib"
|
|
source <(printf '%s\n%s\n' "$role_is_installed_definition" "$remove_shared_ca_if_unused_definition")
|
|
rm -rf "$LIB_DIR/control"
|
|
remove_shared_ca_if_unused
|
|
)
|
|
[[ -f "$role_fixture/etc/control-ca.crt" ]]
|
|
rm -rf "$role_fixture/lib/agent"
|
|
(
|
|
ETC_DIR="$role_fixture/etc"; LIB_DIR="$role_fixture/lib"
|
|
source <(printf '%s\n%s\n' "$role_is_installed_definition" "$remove_shared_ca_if_unused_definition")
|
|
remove_shared_ca_if_unused
|
|
)
|
|
[[ ! -e "$role_fixture/etc/control-ca.crt" ]]
|
|
rm -rf -- "$role_fixture"
|
|
|
|
grep -Fq 'UMask=0077' "$root/deploy/ochenstarik-smm-control.service"
|
|
grep -Fq 'ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/control' "$root/deploy/ochenstarik-smm-control.service"
|
|
native_smoke="$root/tests/bootstrap/run-native-systemd-smoke.sh"
|
|
grep -Fq 'node_code="$(sudo "$system_bootstrap" node-code smoke-node)"' "$native_smoke"
|
|
grep -Fq 'export SMM_ENROLL_CODE="$node_code"' "$native_smoke"
|
|
grep -Fq 'export SMM_ACCEPT_CA_FINGERPRINT=1' "$native_smoke"
|
|
grep -Fq 'sudo test -s /var/lib/ochenstarik-server-monitor-manager/agent/agent.pfx' "$native_smoke"
|
|
grep -Fq 'sudo systemctl is-active --quiet ochenstarik-smm-agent.service' "$native_smoke"
|
|
grep -Fq 'device_code="$(sudo "$system_bootstrap" control-device-code smoke-device)"' "$native_smoke"
|
|
grep -Fq '[[ "$device_code" == SMMDEV1-* ]]' "$native_smoke"
|
|
grep -Fq 'sudo --preserve-env=SMM_ENROLL_CODE,SMM_ACCEPT_CA_FINGERPRINT' "$native_smoke"
|
|
if grep -Fq 'sudo env SMM_ENROLL_CODE=' "$native_smoke"; then
|
|
printf '%s\n' 'native smoke exposes the enrollment code through env argv' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'rm -rf -- "$LIB_DIR/control" "$STATE_DIR/control"' "$bootstrap"
|
|
if grep -Fq 'install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR"' "$bootstrap"; then
|
|
printf '%s\n' 'Agent installation still takes ownership of the shared configuration parent' >&2
|
|
exit 1
|
|
fi
|
|
emergency_help="$(bash "$emergency" --help)"
|
|
grep -Fq 'mesh-disable' <<<"$emergency_help"
|
|
grep -Fq 'firewall-restore' <<<"$emergency_help"
|
|
grep -Fq 'readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"' "$emergency"
|
|
grep -Fq 'chown root:root "$temporary_marker"' "$emergency"
|
|
grep -Fq 'chmod 0600 "$temporary_marker"' "$emergency"
|
|
grep -Fq 'mv -f -- "$temporary_marker" "$RECONCILE_MARKER"' "$emergency"
|
|
grep -Fq '/usr/bin/flock -x 9' "$emergency"
|
|
grep -Fq 'generation="$(</proc/sys/kernel/random/uuid)"' "$emergency"
|
|
[[ "$(grep -Fc ' request_reconciliation' "$emergency")" -ge 2 ]]
|
|
|
|
if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then
|
|
printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>&1; then
|
|
printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2
|
|
exit 1
|
|
fi
|
|
if bash "$emergency" mesh-disable >/dev/null 2>&1; then
|
|
printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2
|
|
exit 1
|
|
fi
|
|
|
|
policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)"
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state"
|
|
printf 'target\t10.77.0.3\tkey-target\tactive\n' >>"$policy_state"
|
|
connect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
bash "$helper" link-connect source target tcp 22 10)"
|
|
grep -Fq 'ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22' <<<"$connect_output"
|
|
grep -Fq 'smm:source:target:tcp:22' <<<"$connect_output"
|
|
disconnect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
bash "$helper" link-disconnect source target tcp 22)"
|
|
grep -Fq 'smm:source:target:tcp:22' <<<"$disconnect_output"
|
|
status_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
bash "$helper" link-status source target tcp 22)"
|
|
[[ "$status_output" == 'disabled' ]] || {
|
|
printf '%s\n' "policy helper returned an invalid factual status" >&2
|
|
exit 1
|
|
}
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
bash "$helper" link-status source target tcp 22 unexpected >/dev/null 2>&1; then
|
|
printf '%s\n' "policy helper unexpectedly accepted extra link-status arguments" >&2
|
|
exit 1
|
|
fi
|
|
|
|
policy_listing="$(mktemp -t smm-policy-listing.XXXXXXXX)"
|
|
cat >"$policy_listing" <<'EOF'
|
|
ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22 counter accept comment "smm:source:target:tcp:22" # handle 5
|
|
counter accept comment "foreign:keep-me" # handle 6
|
|
counter accept comment "smm:source:target:tcp:22" # handle 7
|
|
counter accept comment "smm:FORGED:target:tcp:22" # handle 8
|
|
EOF
|
|
list_error="$(mktemp -t smm-policy-list-error.XXXXXXXX)"
|
|
list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$policy_listing" \
|
|
bash "$helper" link-list 2>"$list_error")"
|
|
[[ "$list_output" == $'source\ttarget\ttcp\t22\nsource\ttarget\ttcp\t22' ]]
|
|
if grep -Fq 'foreign:keep-me' <<<"$list_output"; then
|
|
printf '%s\n' "policy helper exposed a foreign nftables comment" >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'forged managed comment ignored' "$list_error"
|
|
empty_listing="$(mktemp -t smm-policy-empty-listing.XXXXXXXX)"
|
|
empty_list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$empty_listing" \
|
|
bash "$helper" link-list)"
|
|
[[ -z "$empty_list_output" ]]
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_FIREWALL_UNAVAILABLE=1 \
|
|
bash "$helper" link-list >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "missing Link policy table unexpectedly produced a listing" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 79 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'mesh.firewall-unavailable' ]]
|
|
if SMM_POLICY_TESTING=1 bash "$helper" link-list unexpected >/dev/null 2>&1; then
|
|
printf '%s\n' "policy helper unexpectedly accepted extra link-list arguments" >&2
|
|
exit 1
|
|
fi
|
|
inactive_state="$(mktemp -t smm-policy-inactive-state.XXXXXXXX)"
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
printf 'target\t10.77.0.3\t-\treserved\n' >>"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly activated a Link to a reserved Node" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 80 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'mesh.node-not-activated' ]]
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly activated a Link to a missing Node" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 80 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'mesh.node-not-activated' ]]
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
printf 'target\t\tkey-target\tactive\n' >>"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly accepted a blank Node address" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 78 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
printf 'target\tnot-an-ip\tkey-target\tactive\n' >>"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly accepted an invalid Node address" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 78 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
printf 'target\t999.77.0.3\tkey-target\tactive\n' >>"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly accepted an out-of-range Node address" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 78 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
|
printf 'target\t10.77.0.3\tkey-target\tgarbage!\n' >>"$inactive_state"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
|
printf '%s\n' "policy helper unexpectedly accepted a malformed Node status" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 78 ]]
|
|
fi
|
|
[[ "$(<"$list_error")" == 'policy helper: invalid mesh node status: target' ]]
|
|
|
|
generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
|
generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
|
|
reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)"
|
|
printf '%s\n' "$generation_a" >"$reconcile_marker"
|
|
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
|
bash "$helper" reconcile-status)" == "requested:$generation_a" ]]
|
|
printf '%s\n' "$generation_b" >"$reconcile_marker"
|
|
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
|
bash "$helper" reconcile-complete "$generation_a" >/dev/null
|
|
[[ "$(<"$reconcile_marker")" == "$generation_b" ]]
|
|
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
|
bash "$helper" reconcile-complete "$generation_b" >/dev/null
|
|
[[ ! -e "$reconcile_marker" ]]
|
|
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
|
bash "$helper" reconcile-status)" == 'complete' ]]
|
|
missing_mesh="$reconcile_marker-missing/mesh/reconcile-requested"
|
|
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$missing_mesh" \
|
|
bash "$helper" reconcile-status)" == 'complete' ]]
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
|
bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then
|
|
printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2
|
|
exit 1
|
|
fi
|
|
|
|
firewall_error="$(mktemp -t smm-firewall-error.XXXXXXXX)"
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
SMM_POLICY_FIREWALL_UNAVAILABLE=1 bash "$helper" \
|
|
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
|
|
printf '%s\n' "missing firewall unexpectedly produced a factual Link status" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 79 ]]
|
|
fi
|
|
[[ "$(<"$firewall_error")" == 'mesh.firewall-unavailable' ]]
|
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|
SMM_POLICY_FIREWALL_ERROR='permission denied' bash "$helper" \
|
|
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
|
|
printf '%s\n' "unknown nft inspection error unexpectedly produced a factual Link status" >&2
|
|
exit 1
|
|
else
|
|
[[ $? -eq 78 ]]
|
|
fi
|
|
grep -Fq 'permission denied' "$firewall_error"
|
|
if grep -Fq 'mesh.firewall-unavailable' "$firewall_error"; then
|
|
printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2
|
|
exit 1
|
|
fi
|
|
rm -f -- "$firewall_error" "$reconcile_marker" "$policy_state" \
|
|
"$policy_listing" "$empty_listing" "$list_error" "$inactive_state"
|
|
|
|
extract_shell_function() {
|
|
local name="$1"
|
|
awk -v signature="$name() {" '
|
|
$0 == signature { emitting = 1 }
|
|
emitting { print }
|
|
emitting && $0 == "}" { exit }
|
|
' "$acceptance"
|
|
}
|
|
eval "$(extract_shell_function probe_factual_status)"
|
|
SOURCE_NODE_ID=source
|
|
TARGET_PORT=22
|
|
probe_counter="$(mktemp -t smm-factual-probe.XXXXXXXX)"
|
|
printf '%s\n' 0 >"$probe_counter"
|
|
hub_ssh() {
|
|
local count
|
|
count="$(( $(<"$probe_counter") + 1 ))"
|
|
printf '%s\n' "$count" >"$probe_counter"
|
|
[[ "$count" -eq 1 ]] && printf '%s\n' disabled || printf '%s\n' active
|
|
}
|
|
if probe_factual_status target active; then
|
|
printf '%s\n' "initial factual mismatch unexpectedly passed" >&2
|
|
exit 1
|
|
fi
|
|
probe_factual_status target active || {
|
|
printf '%s\n' "factual probe did not allow convergence after an initial mismatch" >&2
|
|
exit 1
|
|
}
|
|
rm -f -- "$probe_counter" "${reconcile_marker}.lock"
|
|
|
|
fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)"
|
|
trap 'rm -rf -- "$fixture"' EXIT
|
|
mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap"
|
|
install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent"
|
|
install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control"
|
|
install -m 0755 /bin/true "$fixture/payload/provisioning-helper/ochenstarik-smm-provisioning-helper"
|
|
install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply"
|
|
install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency"
|
|
install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/"
|
|
install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/"
|
|
install -m 0644 "$root/deploy/ochenstarik-smm-provisioning-helper.service" "$fixture/payload/deploy/"
|
|
install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/"
|
|
install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh"
|
|
tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control provisioning-helper deploy bootstrap
|
|
sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256"
|
|
|
|
# Create a signed manifest for verify-release (required by hardened verify_archive)
|
|
release_hash="$(sha256sum "$fixture/release.tar.gz" | awk '{ print $1 }')"
|
|
cat >"$fixture/server-monitor-manager-manifest.json" <<MEOF
|
|
{
|
|
"version": "v0.0.0-test",
|
|
"hashes": {
|
|
"release.tar.gz": "$release_hash"
|
|
}
|
|
}
|
|
MEOF
|
|
|
|
if command -v cosign &>/dev/null; then
|
|
COSIGN_PASSWORD="" cosign generate-key-pair --output-key-prefix="$fixture/contract-test"
|
|
COSIGN_PASSWORD="" cosign sign-blob --yes --key "$fixture/contract-test.key" \
|
|
--output-signature "$fixture/server-monitor-manager-manifest.sig" \
|
|
"$fixture/server-monitor-manager-manifest.json"
|
|
SMM_TEST_PUBKEY="$fixture/contract-test.pub" \
|
|
bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null
|
|
else
|
|
# Cosign unavailable — fall back to SMM_ALLOW_UNSIGNED for contract test only
|
|
SMM_ALLOW_UNSIGNED=1 bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null
|
|
fi
|
|
|
|
# Corrupt the manifest hash and verify rejection
|
|
cat >"$fixture/server-monitor-manager-manifest.json" <<MEOF
|
|
{
|
|
"version": "v0.0.0-test",
|
|
"hashes": {
|
|
"release.tar.gz": "$(printf '%064d' 0)"
|
|
}
|
|
}
|
|
MEOF
|
|
if command -v cosign &>/dev/null; then
|
|
COSIGN_PASSWORD="" cosign sign-blob --yes --key "$fixture/contract-test.key" \
|
|
--output-signature "$fixture/server-monitor-manager-manifest.sig" \
|
|
"$fixture/server-monitor-manager-manifest.json"
|
|
if SMM_TEST_PUBKEY="$fixture/contract-test.pub" \
|
|
bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null 2>&1; then
|
|
printf '%s\n' "corrupt release checksum unexpectedly succeeded" >&2
|
|
exit 1
|
|
fi
|
|
else
|
|
printf '%064d %s\n' 0 release.tar.gz >"$fixture/release.tar.gz.sha256"
|
|
if SMM_ALLOW_UNSIGNED=1 bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null 2>&1; then
|
|
printf '%s\n' "corrupt release checksum unexpectedly succeeded" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Version ordering. Lexicographic comparison would rank "0.1.0-alpha.10" below
|
|
# "0.1.0-alpha.9" and reject every release after the ninth as a downgrade.
|
|
version_helpers="$(sed -n '/^normalize_version()/,/^}/p;/^version_lt()/,/^}/p' "$bootstrap")"
|
|
version_case() {
|
|
local left="$1" right="$2" expected="$3" actual
|
|
if bash -c "$version_helpers
|
|
version_lt \"\$1\" \"\$2\"" _ "$left" "$right"; then actual="lt"; else actual="ge"; fi
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
printf 'version_lt %s %s returned %s, expected %s\n' "$left" "$right" "$actual" "$expected" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
version_case "v0.1.0-alpha.9" "v0.1.0-alpha.10" lt
|
|
version_case "v0.1.0-alpha.10" "v0.1.0-alpha.9" ge
|
|
version_case "v0.1.0-alpha.9" "v0.1.0-alpha.9" ge
|
|
version_case "0.1.0-alpha.9" "v0.1.0-alpha.9" ge
|
|
version_case "v0.1.0-alpha.9" "v0.2.0" lt
|
|
version_case "v0.2.0" "v0.1.0-alpha.9" ge
|
|
|
|
# The downgrade guard must compare against the recorded installed version, never against
|
|
# PROGRAM_VERSION: that constant tracks the source tree, not the deployed component.
|
|
grep -Fq 'installed="$(read_installed_version "$role")"' "$bootstrap" || {
|
|
printf '%s\n' "update_role must compare against the recorded installed version" >&2
|
|
exit 1
|
|
}
|
|
if grep -qE '"\$new_version" *< *"\$PROGRAM_VERSION"' "$bootstrap"; then
|
|
printf '%s\n' "lexicographic version comparison against PROGRAM_VERSION reintroduced" >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'record_installed_version control' "$bootstrap" || {
|
|
printf '%s\n' "install-control must record the installed version" >&2
|
|
exit 1
|
|
}
|
|
grep -Fq 'record_installed_version agent' "$bootstrap" || {
|
|
printf '%s\n' "install-agent must record the installed version" >&2
|
|
exit 1
|
|
}
|
|
|
|
printf '%s\n' "BOOTSTRAP_CONTRACT=PASS"
|
|
|