#!/usr/bin/env bash set -Eeuo pipefail IFS=$'\n\t' root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" helper="$root/deploy/ochenstarik-smm-policy-apply" emergency="$root/deploy/ochenstarik-smm-emergency" acceptance="$root/tests/acceptance/three-server-mesh.sh" grep -Fq 'listing="$(/usr/sbin/nft -a list chain' "$helper" || { printf '%s\n' "policy status probe must fail closed when nftables cannot be inspected" >&2 exit 1 } grep -Fq "grep -Eiq 'No such file or directory|does not exist'" "$helper" provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service" grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit" grep -Fq 'ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback' "$provisioning_helper_unit" grep -Fq 'install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback"' "$bootstrap" if grep -Fq 'SMM_EnrollToken=$ENROLL_TOKEN' "$bootstrap"; then printf '%s\n' "enrollment token is exposed through process argv" >&2 exit 1 fi grep -Fq 'readonly ENROLLMENT_DIR="${STATE_DIR}-enrollment"' "$bootstrap" grep -Fq 'install -d -m 0710 -o root -g "$AGENT_USER" "$ENROLLMENT_DIR"' "$bootstrap" grep -Fq 'token_temp="$(mktemp "$ENROLLMENT_DIR/.enroll-token.XXXXXXXX")"' "$bootstrap" if grep -Fq '$STATE_DIR/enrollment' "$bootstrap"; then printf '%s\n' "enrollment directory is beneath Control-writable state" >&2 exit 1 fi grep -Fq 'chmod 0400 "$token_temp"' "$bootstrap" grep -Fq 'mv -fT -- "$token_temp" "$token_file"' "$bootstrap" grep -Fq '"SMM_EnrollTokenFile=$token_file"' "$bootstrap" grep -Fq 'rm -f -- "$token_file"' "$bootstrap" grep -Fq 'rm -f -- "$ENROLLMENT_TOKEN_FILE"' "$bootstrap" grep -Fq 'rm -f -- "$ENROLLMENT_TOKEN_TEMP"' "$bootstrap" grep -Fq 'SMM_AgentUid=$(id -u "$AGENT_USER")' "$bootstrap" grep -Fq 'refresh_agent_uid() {' "$bootstrap" grep -Fq 'agent_uid="$(id -u "$AGENT_USER")"' "$bootstrap" grep -Fq "printf 'SMM_AgentUid=%s\\n' \"\$agent_uid\"" "$bootstrap" grep -Fq ' refresh_agent_uid' "$bootstrap" grep -Fq 'temp="$(mktemp "$ETC_DIR/.agent.env.XXXXXXXX")"' "$bootstrap" grep -Fq 'mv -fT -- "$temp" "$env_file"' "$bootstrap" refresh_line="$(grep -F -m1 -n ' refresh_agent_uid' "$bootstrap" | cut -d: -f1)" stop_line="$(grep -F -m1 -n ' systemctl stop "$unit"' "$bootstrap" | cut -d: -f1)" (( refresh_line < stop_line )) help_output="$(bash "$bootstrap" --help)" version_output="$(bash "$bootstrap" --version)" grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output" grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output" grep -Fq "install-node ARCHIVE" <<<"$help_output" grep -Fq "mesh-init PUBLIC_ENDPOINT" <<<"$help_output" grep -Fq "peer-add SMMPEER1_CODE" <<<"$help_output" grep -Fq "mesh-status" <<<"$help_output" grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output" grep -Fq "node-code NODE_ID" <<<"$help_output" grep -Fq "verify-release ARCHIVE" <<<"$help_output" grep -Fq "node-token NODE_ID" <<<"$help_output" grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" emergency_help="$(bash "$emergency" --help)" grep -Fq 'mesh-disable' <<<"$emergency_help" grep -Fq 'firewall-restore' <<<"$emergency_help" grep -Fq 'readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"' "$emergency" grep -Fq 'chown root:root "$temporary_marker"' "$emergency" grep -Fq 'chmod 0600 "$temporary_marker"' "$emergency" grep -Fq 'mv -f -- "$temporary_marker" "$RECONCILE_MARKER"' "$emergency" grep -Fq '/usr/bin/flock -x 9' "$emergency" grep -Fq 'generation="$(/dev/null 2>&1; then printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2 exit 1 fi if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>&1; then printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2 exit 1 fi if bash "$emergency" mesh-disable >/dev/null 2>&1; then printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2 exit 1 fi policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)" printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state" printf 'target\t10.77.0.3\tkey-target\tactive\n' >>"$policy_state" connect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ bash "$helper" link-connect source target tcp 22 10)" grep -Fq 'ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22' <<<"$connect_output" grep -Fq 'smm:source:target:tcp:22' <<<"$connect_output" disconnect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ bash "$helper" link-disconnect source target tcp 22)" grep -Fq 'smm:source:target:tcp:22' <<<"$disconnect_output" status_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ bash "$helper" link-status source target tcp 22)" [[ "$status_output" == 'disabled' ]] || { printf '%s\n' "policy helper returned an invalid factual status" >&2 exit 1 } if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ bash "$helper" link-status source target tcp 22 unexpected >/dev/null 2>&1; then printf '%s\n' "policy helper unexpectedly accepted extra link-status arguments" >&2 exit 1 fi policy_listing="$(mktemp -t smm-policy-listing.XXXXXXXX)" cat >"$policy_listing" <<'EOF' ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22 counter accept comment "smm:source:target:tcp:22" # handle 5 counter accept comment "foreign:keep-me" # handle 6 counter accept comment "smm:source:target:tcp:22" # handle 7 counter accept comment "smm:FORGED:target:tcp:22" # handle 8 EOF list_error="$(mktemp -t smm-policy-list-error.XXXXXXXX)" list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$policy_listing" \ bash "$helper" link-list 2>"$list_error")" [[ "$list_output" == $'source\ttarget\ttcp\t22\nsource\ttarget\ttcp\t22' ]] if grep -Fq 'foreign:keep-me' <<<"$list_output"; then printf '%s\n' "policy helper exposed a foreign nftables comment" >&2 exit 1 fi grep -Fq 'forged managed comment ignored' "$list_error" empty_listing="$(mktemp -t smm-policy-empty-listing.XXXXXXXX)" empty_list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$empty_listing" \ bash "$helper" link-list)" [[ -z "$empty_list_output" ]] if SMM_POLICY_TESTING=1 SMM_POLICY_FIREWALL_UNAVAILABLE=1 \ bash "$helper" link-list >/dev/null 2>"$list_error"; then printf '%s\n' "missing Link policy table unexpectedly produced a listing" >&2 exit 1 else [[ $? -eq 79 ]] fi [[ "$(<"$list_error")" == 'mesh.firewall-unavailable' ]] if SMM_POLICY_TESTING=1 bash "$helper" link-list unexpected >/dev/null 2>&1; then printf '%s\n' "policy helper unexpectedly accepted extra link-list arguments" >&2 exit 1 fi inactive_state="$(mktemp -t smm-policy-inactive-state.XXXXXXXX)" printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" printf 'target\t10.77.0.3\t-\treserved\n' >>"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly activated a Link to a reserved Node" >&2 exit 1 else [[ $? -eq 80 ]] fi [[ "$(<"$list_error")" == 'mesh.node-not-activated' ]] printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly activated a Link to a missing Node" >&2 exit 1 else [[ $? -eq 80 ]] fi [[ "$(<"$list_error")" == 'mesh.node-not-activated' ]] printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" printf 'target\t\tkey-target\tactive\n' >>"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly accepted a blank Node address" >&2 exit 1 else [[ $? -eq 78 ]] fi [[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]] printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" printf 'target\tnot-an-ip\tkey-target\tactive\n' >>"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly accepted an invalid Node address" >&2 exit 1 else [[ $? -eq 78 ]] fi [[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]] printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" printf 'target\t999.77.0.3\tkey-target\tactive\n' >>"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly accepted an out-of-range Node address" >&2 exit 1 else [[ $? -eq 78 ]] fi [[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]] printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state" printf 'target\t10.77.0.3\tkey-target\tgarbage!\n' >>"$inactive_state" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \ bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then printf '%s\n' "policy helper unexpectedly accepted a malformed Node status" >&2 exit 1 else [[ $? -eq 78 ]] fi [[ "$(<"$list_error")" == 'policy helper: invalid mesh node status: target' ]] generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)" printf '%s\n' "$generation_a" >"$reconcile_marker" [[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \ bash "$helper" reconcile-status)" == "requested:$generation_a" ]] printf '%s\n' "$generation_b" >"$reconcile_marker" SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \ bash "$helper" reconcile-complete "$generation_a" >/dev/null [[ "$(<"$reconcile_marker")" == "$generation_b" ]] SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \ bash "$helper" reconcile-complete "$generation_b" >/dev/null [[ ! -e "$reconcile_marker" ]] [[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \ bash "$helper" reconcile-status)" == 'complete' ]] missing_mesh="$reconcile_marker-missing/mesh/reconcile-requested" [[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$missing_mesh" \ bash "$helper" reconcile-status)" == 'complete' ]] if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \ bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2 exit 1 fi firewall_error="$(mktemp -t smm-firewall-error.XXXXXXXX)" if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ SMM_POLICY_FIREWALL_UNAVAILABLE=1 bash "$helper" \ link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then printf '%s\n' "missing firewall unexpectedly produced a factual Link status" >&2 exit 1 else [[ $? -eq 79 ]] fi [[ "$(<"$firewall_error")" == 'mesh.firewall-unavailable' ]] if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ SMM_POLICY_FIREWALL_ERROR='permission denied' bash "$helper" \ link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then printf '%s\n' "unknown nft inspection error unexpectedly produced a factual Link status" >&2 exit 1 else [[ $? -eq 78 ]] fi grep -Fq 'permission denied' "$firewall_error" if grep -Fq 'mesh.firewall-unavailable' "$firewall_error"; then printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2 exit 1 fi rm -f -- "$firewall_error" "$reconcile_marker" "$policy_state" \ "$policy_listing" "$empty_listing" "$list_error" "$inactive_state" extract_shell_function() { local name="$1" awk -v signature="$name() {" ' $0 == signature { emitting = 1 } emitting { print } emitting && $0 == "}" { exit } ' "$acceptance" } eval "$(extract_shell_function probe_factual_status)" SOURCE_NODE_ID=source TARGET_PORT=22 probe_counter="$(mktemp -t smm-factual-probe.XXXXXXXX)" printf '%s\n' 0 >"$probe_counter" hub_ssh() { local count count="$(( $(<"$probe_counter") + 1 ))" printf '%s\n' "$count" >"$probe_counter" [[ "$count" -eq 1 ]] && printf '%s\n' disabled || printf '%s\n' active } if probe_factual_status target active; then printf '%s\n' "initial factual mismatch unexpectedly passed" >&2 exit 1 fi probe_factual_status target active || { printf '%s\n' "factual probe did not allow convergence after an initial mismatch" >&2 exit 1 } rm -f -- "$probe_counter" "${reconcile_marker}.lock" fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)" trap 'rm -rf -- "$fixture"' EXIT mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap" install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent" install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" install -m 0755 /bin/true "$fixture/payload/provisioning-helper/ochenstarik-smm-provisioning-helper" install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency" install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-provisioning-helper.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/" install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh" tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control provisioning-helper deploy bootstrap sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256" bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null printf '%064d %s\n' 0 release.tar.gz >"$fixture/release.tar.gz.sha256" if bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null 2>&1; then printf '%s\n' "corrupt release checksum unexpectedly succeeded" >&2 exit 1 fi printf '%s\n' "BOOTSTRAP_CONTRACT=PASS"