From d9ca16b27e2ba6816fb9d86e5f44213668cca454 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:15:45 +0700 Subject: [PATCH 01/31] Add standalone Linux bootstrap foundation --- .github/workflows/linux-control-agent.yml | 8 + .github/workflows/linux-release.yml | 55 ++- README.md | 2 +- deploy/ochenstarik-server-monitor-manager.sh | 486 +++++++++++++++++++ deploy/ochenstarik-smm-policy-apply | 27 ++ docs/linux-bootstrap.md | 83 ++++ tests/bootstrap/test-bootstrap-contract.sh | 28 ++ 7 files changed, 687 insertions(+), 2 deletions(-) create mode 100755 deploy/ochenstarik-server-monitor-manager.sh create mode 100755 deploy/ochenstarik-smm-policy-apply create mode 100644 docs/linux-bootstrap.md create mode 100755 tests/bootstrap/test-bootstrap-contract.sh diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index f9b52ab..c860720 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -37,6 +37,14 @@ jobs: bash -n tests/acceptance/three-server-mesh.sh shellcheck --severity=error tests/acceptance/three-server-mesh.sh + - name: Verify standalone bootstrap + run: | + bash -n deploy/ochenstarik-server-monitor-manager.sh + bash -n deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh + shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + bash tests/bootstrap/test-bootstrap-contract.sh + - name: Publish agent amd64 run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-x64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index 87475c1..3b81bd2 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -10,6 +10,54 @@ permissions: contents: write jobs: + bootstrap: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Validate bootstrap + run: | + bash -n deploy/ochenstarik-server-monitor-manager.sh + bash -n deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh + shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + bash tests/bootstrap/test-bootstrap-contract.sh + + - name: Package bootstrap + shell: bash + run: | + set -Eeuo pipefail + install -m 0755 deploy/ochenstarik-server-monitor-manager.sh ochenstarik-server-monitor-manager.sh + sha256sum ochenstarik-server-monitor-manager.sh > ochenstarik-server-monitor-manager.sh.sha256 + bootstrap_sha="$(sha256sum ochenstarik-server-monitor-manager.sh | awk '{print $1}')" + jq -n \ + --arg schema "smm-bootstrap-manifest/v1" \ + --arg version "${GITHUB_REF_NAME}" \ + --arg bootstrap "ochenstarik-server-monitor-manager.sh" \ + --arg bootstrap_sha256 "$bootstrap_sha" \ + '{schema: $schema, version: $version, bootstrap: $bootstrap, bootstrap_sha256: $bootstrap_sha256, supported_runtimes: ["linux-x64", "linux-arm64"]}' \ + > server-monitor-manager-bootstrap-manifest.json + + - name: Upload bootstrap artifact + uses: actions/upload-artifact@v6 + with: + name: server-monitor-manager-bootstrap + path: | + ochenstarik-server-monitor-manager.sh + ochenstarik-server-monitor-manager.sh.sha256 + server-monitor-manager-bootstrap-manifest.json + + - name: Attach bootstrap to GitHub Release + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v2 + with: + prerelease: ${{ contains(github.ref_name, '-') }} + files: | + ochenstarik-server-monitor-manager.sh + ochenstarik-server-monitor-manager.sh.sha256 + server-monitor-manager-bootstrap-manifest.json + publish: runs-on: ubuntu-latest strategy: @@ -34,8 +82,13 @@ jobs: shell: bash run: | set -Eeuo pipefail + install -d out/deploy out/bootstrap + install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ + install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ + install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz" - tar -C out -czf "$archive" agent control + tar -C out -czf "$archive" agent control deploy bootstrap sha256sum "$archive" > "$archive.sha256" - name: Upload artifact diff --git a/README.md b/README.md index 57b626f..0099910 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The control plane and data plane are separated: See [architecture](docs/architecture.md), [security model](docs/security-model.md), [roadmap](docs/roadmap.md), [Linux bootstrap contract](docs/installer-contract.md), and the [Provisioning and Xray specification](docs/provisioning-vpn-requirements.md). -Operational procedures are documented in [Control backup and recovery](docs/control-backup.md) and the [three-server acceptance test](docs/three-server-acceptance.md). +Operational procedures are documented in [Linux bootstrap](docs/linux-bootstrap.md), [Control backup and recovery](docs/control-backup.md), and the [three-server acceptance test](docs/three-server-acceptance.md). ## Repository layout diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh new file mode 100755 index 0000000..d556f3c --- /dev/null +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -0,0 +1,486 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +readonly PROGRAM="ochenstarik-server-monitor-manager" +readonly VERSION="0.2.0-dev" +readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager" +readonly LIB_DIR="/usr/local/lib/ochenstarik-server-monitor-manager" +readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager" +readonly BACKUP_DIR="${STATE_DIR}/bootstrap-backups" +readonly CONTROL_USER="ochenstarik-smm-control" +readonly AGENT_USER="ochenstarik-smm-agent" +readonly CONTROL_UNIT="ochenstarik-smm-control.service" +readonly AGENT_UNIT="ochenstarik-smm-agent.service" +readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply" +readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control" + +TEMP_DIR="" + +log() { printf '%s\n' "[$PROGRAM] $*"; } +fail() { printf '%s\n' "[$PROGRAM] ERROR: $*" >&2; exit 1; } + +cleanup() { + if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then + rm -rf -- "$TEMP_DIR" + fi +} +trap cleanup EXIT + +usage() { + cat <<'EOF' +Server Monitor Manager Linux bootstrap + +Usage: + ochenstarik-server-monitor-manager.sh preflight + ochenstarik-server-monitor-manager.sh install-control ARCHIVE PUBLIC_HOST [HTTPS_PORT] + ochenstarik-server-monitor-manager.sh install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT + ochenstarik-server-monitor-manager.sh update-control ARCHIVE + ochenstarik-server-monitor-manager.sh update-agent ARCHIVE + ochenstarik-server-monitor-manager.sh rollback control|agent [BACKUP_ID] + ochenstarik-server-monitor-manager.sh node-token NODE_ID + ochenstarik-server-monitor-manager.sh control-ca-fingerprint + ochenstarik-server-monitor-manager.sh status + ochenstarik-server-monitor-manager.sh uninstall-agent [--purge] + ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control + ochenstarik-server-monitor-manager.sh version + +ARCHIVE must have a matching ARCHIVE.sha256 file. Agent enrollment reads the +one-time token from SMM_ENROLL_TOKEN or from a hidden local prompt; it is never +written to agent.env. +EOF +} + +require_root() { + [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "This action must run as root (use sudo)." +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "Required command is missing: $1" +} + +validate_platform() { + [[ -r /etc/os-release ]] || fail "/etc/os-release is missing." + # shellcheck disable=SC1091 + . /etc/os-release + case "${ID:-}" in + ubuntu) + case "${VERSION_ID:-}" in 22.04|24.04) ;; *) fail "Unsupported Ubuntu version: ${VERSION_ID:-unknown}" ;; esac + ;; + debian) + case "${VERSION_ID:-}" in 12|13) ;; *) fail "Unsupported Debian version: ${VERSION_ID:-unknown}" ;; esac + ;; + *) fail "Unsupported distribution: ${ID:-unknown}" ;; + esac + case "$(uname -m)" in + x86_64|aarch64|arm64) ;; + *) fail "Unsupported architecture: $(uname -m)" ;; + esac + [[ "$(ps -p 1 -o comm=)" == "systemd" ]] || fail "systemd must be PID 1." +} + +validate_node_id() { + [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]] \ + || fail "Node id must contain 1-63 lowercase letters, digits, or hyphens." +} + +validate_port() { + [[ "$1" =~ ^[0-9]+$ ]] && (( 10#$1 >= 1 && 10#$1 <= 65535 )) \ + || fail "Port must be in range 1-65535." +} + +validate_control_url() { + [[ "$1" =~ ^https://[A-Za-z0-9._:\[\]-]+(:[0-9]{1,5})?/?$ ]] \ + || fail "Control URL must be an https URL without a path or credentials." +} + +verify_archive() { + local archive="$1" checksum_file expected actual entry + [[ -f "$archive" ]] || fail "Archive not found: $archive" + checksum_file="${archive}.sha256" + [[ -f "$checksum_file" ]] || fail "Checksum file not found: $checksum_file" + expected="$(awk 'NR == 1 { print $1 }' "$checksum_file")" + [[ "$expected" =~ ^[0-9a-fA-F]{64}$ ]] || fail "Invalid checksum file: $checksum_file" + actual="$(sha256sum "$archive" | awk '{ print $1 }')" + [[ "${actual,,}" == "${expected,,}" ]] || fail "Archive checksum mismatch." + + while IFS= read -r entry; do + [[ -n "$entry" ]] || continue + [[ "$entry" != /* && "$entry" != *".."* ]] || fail "Unsafe archive entry: $entry" + case "$entry" in + agent|agent/*|control|control/*|deploy|deploy/*|bootstrap|bootstrap/*) ;; + *) fail "Unexpected archive entry: $entry" ;; + esac + done < <(tar -tzf "$archive") +} + +extract_archive() { + local archive="$1" + verify_archive "$archive" + TEMP_DIR="$(mktemp -d -t smm-bootstrap.XXXXXXXX)" + chmod 700 "$TEMP_DIR" + tar -xzf "$archive" -C "$TEMP_DIR" --no-same-owner --no-same-permissions + [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive." + [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." +} + +ensure_system_user() { + local user="$1" + if ! getent group "$user" >/dev/null; then + groupadd --system "$user" + fi + if ! id "$user" >/dev/null 2>&1; then + useradd --system --gid "$user" --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin "$user" + fi +} + +install_tree_atomic() { + local source="$1" destination="$2" owner="$3" staging + [[ -d "$source" ]] || fail "Release payload is missing: $source" + staging="${destination}.new.$$" + rm -rf -- "$staging" + install -d -m 0755 "$staging" + cp -a -- "$source/." "$staging/" + chown -R "$owner" "$staging" + find "$staging" -type d -exec chmod 0755 {} + + find "$staging" -type f -exec chmod 0644 {} + + find "$staging" -type f -name 'ochenstarik-smm-*' -exec chmod 0755 {} + + rm -rf -- "$destination" + mv -- "$staging" "$destination" +} + +create_backup() { + local role="$1" backup_id archive list=() + backup_id="$(date -u +%Y%m%dT%H%M%SZ)-${role}-$RANDOM" + install -d -m 0700 "$BACKUP_DIR" + archive="$BACKUP_DIR/${backup_id}.tar.gz" + case "$role" in + control) + list=( + "usr/local/lib/ochenstarik-server-monitor-manager/control" + "etc/ochenstarik-server-monitor-manager/control.env" + "etc/ochenstarik-server-monitor-manager/control-ca.pfx" + "etc/ochenstarik-server-monitor-manager/control-server.pfx" + "etc/systemd/system/$CONTROL_UNIT" + "usr/local/libexec/ochenstarik-smm-policy-apply" + "etc/sudoers.d/ochenstarik-smm-control" + ) + ;; + agent) + list=( + "usr/local/lib/ochenstarik-server-monitor-manager/agent" + "etc/ochenstarik-server-monitor-manager/agent.env" + "etc/ochenstarik-server-monitor-manager/control-ca.crt" + "etc/systemd/system/$AGENT_UNIT" + ) + ;; + *) fail "Unknown backup role: $role" ;; + esac + local existing=() item + for item in "${list[@]}"; do + [[ -e "/$item" ]] && existing+=("$item") + done + if (( ${#existing[@]} == 0 )); then + printf '%s\n' "empty" >"$BACKUP_DIR/${backup_id}.empty" + else + tar -C / -czf "$archive" -- "${existing[@]}" + chmod 0600 "$archive" + fi + printf '%s\n' "$backup_id" +} + +install_unit() { + local source="$1" unit="$2" + install -m 0644 "$source" "/etc/systemd/system/$unit" + systemctl daemon-reload +} + +create_control_certificates() { + local public_host="$1" ca_key ca_cert serial_file server_key server_csr ext_file san + ca_key="$TEMP_DIR/control-ca.key" + ca_cert="$TEMP_DIR/control-ca.crt" + serial_file="$TEMP_DIR/control-ca.srl" + server_key="$TEMP_DIR/control-server.key" + server_csr="$TEMP_DIR/control-server.csr" + ext_file="$TEMP_DIR/control-server.ext" + if [[ "$public_host" == *:* || "$public_host" =~ ^[0-9]+(\.[0-9]+){3}$ ]]; then + san="IP:$public_host" + else + [[ "$public_host" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$ ]] \ + || fail "Invalid public host name." + san="DNS:$public_host" + fi + + openssl ecparam -name prime256v1 -genkey -noout -out "$ca_key" + openssl req -x509 -new -sha256 -days 3650 -key "$ca_key" -out "$ca_cert" \ + -subj "/CN=Server Monitor Manager Control CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" + openssl ecparam -name prime256v1 -genkey -noout -out "$server_key" + openssl req -new -sha256 -key "$server_key" -out "$server_csr" \ + -subj "/CN=$public_host" + printf '%s\n' \ + "basicConstraints=critical,CA:FALSE" \ + "keyUsage=critical,digitalSignature,keyEncipherment" \ + "extendedKeyUsage=serverAuth" \ + "subjectAltName=$san" >"$ext_file" + openssl x509 -req -sha256 -days 825 -in "$server_csr" -CA "$ca_cert" -CAkey "$ca_key" \ + -CAserial "$serial_file" -CAcreateserial -out "$TEMP_DIR/control-server.crt" -extfile "$ext_file" + openssl pkcs12 -export -out "$ETC_DIR/control-ca.pfx" -inkey "$ca_key" -in "$ca_cert" -passout pass: + openssl pkcs12 -export -out "$ETC_DIR/control-server.pfx" -inkey "$server_key" \ + -in "$TEMP_DIR/control-server.crt" -certfile "$ca_cert" -passout pass: + install -m 0644 "$ca_cert" "$ETC_DIR/control-ca.crt" + chown root:"$CONTROL_USER" "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx" + chmod 0640 "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx" +} + +install_control() { + local archive="$1" public_host="$2" port="${3:-7443}" backup_id + require_root + validate_platform + validate_port "$port" + require_command openssl + require_command sha256sum + require_command tar + require_command systemctl + require_command sudo + require_command visudo + extract_archive "$archive" + [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." + backup_id="$(create_backup control)" + ensure_system_user "$CONTROL_USER" + install -d -m 0750 -o root -g "$CONTROL_USER" "$ETC_DIR" + install -d -m 0750 -o "$CONTROL_USER" -g "$CONTROL_USER" "$STATE_DIR" "$STATE_DIR/backups" + install_tree_atomic "$TEMP_DIR/control" "$LIB_DIR/control" "root:root" + if [[ ! -f "$ETC_DIR/control-ca.pfx" || ! -f "$ETC_DIR/control-server.pfx" ]]; then + create_control_certificates "$public_host" + fi + cat >"$ETC_DIR/control.env" <"$SUDOERS_FILE" + chmod 0440 "$SUDOERS_FILE" + visudo -cf "$SUDOERS_FILE" >/dev/null + install_unit "$TEMP_DIR/deploy/$CONTROL_UNIT" "$CONTROL_UNIT" + systemctl enable --now "$CONTROL_UNIT" + systemctl is-active --quiet "$CONTROL_UNIT" || { + systemctl status --no-pager "$CONTROL_UNIT" >&2 || true + fail "Control service failed; backup is $backup_id" + } + log "Control installed. Backup: $backup_id" + log "CA fingerprint: $(openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256 | cut -d= -f2)" +} + +read_enrollment_token() { + if [[ -n "${SMM_ENROLL_TOKEN:-}" ]]; then + ENROLL_TOKEN="$SMM_ENROLL_TOKEN" + unset SMM_ENROLL_TOKEN + return + fi + [[ -t 0 ]] || fail "Set SMM_ENROLL_TOKEN or run from an interactive local terminal." + read -r -s -p "One-time enrollment token: " ENROLL_TOKEN + printf '\n' + [[ -n "$ENROLL_TOKEN" ]] || fail "Enrollment token is empty." +} + +install_agent() { + local archive="$1" node_id="$2" control_url="$3" ca_cert="$4" backup_id + require_root + validate_platform + validate_node_id "$node_id" + validate_control_url "$control_url" + [[ -f "$ca_cert" ]] || fail "Control CA certificate not found: $ca_cert" + require_command sha256sum + require_command tar + require_command systemctl + require_command runuser + require_command openssl + openssl x509 -in "$ca_cert" -noout >/dev/null 2>&1 || fail "Invalid Control CA certificate." + extract_archive "$archive" + [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." + backup_id="$(create_backup agent)" + ensure_system_user "$AGENT_USER" + install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR" + install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent" + install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root" + if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then + install -m 0600 -o "$AGENT_USER" -g "$AGENT_USER" "$ca_cert" "$ETC_DIR/control-ca.crt" + else + chown "$AGENT_USER:$AGENT_USER" "$ETC_DIR/control-ca.crt" + chmod 0600 "$ETC_DIR/control-ca.crt" + fi + cat >"$ETC_DIR/agent.env" <&2 || true + fail "Agent service failed; backup is $backup_id" + } + log "Agent $node_id installed and enrolled. Backup: $backup_id" +} + +update_role() { + local role="$1" archive="$2" binary unit user backup_id + require_root + validate_platform + extract_archive "$archive" + case "$role" in + control) binary="ochenstarik-smm-control"; unit="$CONTROL_UNIT"; user="root:root" ;; + agent) binary="ochenstarik-smm-agent"; unit="$AGENT_UNIT"; user="root:root" ;; + *) fail "Unknown role: $role" ;; + esac + [[ -x "$TEMP_DIR/$role/$binary" ]] || fail "$role binary is missing." + systemctl stop "$unit" + backup_id="$(create_backup "$role")" + install_tree_atomic "$TEMP_DIR/$role" "$LIB_DIR/$role" "$user" + systemctl start "$unit" + if ! systemctl is-active --quiet "$unit"; then + log "Update failed; restoring backup $backup_id" + restore_backup "$role" "$backup_id" + fail "$role update was rolled back." + fi + log "$role updated. Backup: $backup_id" +} + +latest_backup_id() { + local role="$1" path + path="$(find "$BACKUP_DIR" -maxdepth 1 -type f \( -name "*-${role}-*.tar.gz" -o -name "*-${role}-*.empty" \) -printf '%f\n' 2>/dev/null | sort | tail -n1)" + [[ -n "$path" ]] || fail "No backup found for $role." + printf '%s\n' "${path%.tar.gz}" | sed 's/\.empty$//' +} + +restore_backup() { + local role="$1" backup_id="$2" archive="$BACKUP_DIR/${backup_id}.tar.gz" unit + case "$role" in control) unit="$CONTROL_UNIT" ;; agent) unit="$AGENT_UNIT" ;; *) fail "Unknown role: $role" ;; esac + [[ -f "$archive" ]] || fail "Backup archive not found: $backup_id" + systemctl stop "$unit" || true + tar -C / -xzf "$archive" + systemctl daemon-reload + systemctl start "$unit" + systemctl is-active --quiet "$unit" || fail "Rollback restored files but service is not active." + log "$role restored from $backup_id" +} + +rollback_role() { + local role="$1" backup_id="${2:-}" + require_root + [[ -n "$backup_id" ]] || backup_id="$(latest_backup_id "$role")" + restore_backup "$role" "$backup_id" +} + +show_status() { + local unit + for unit in "$CONTROL_UNIT" "$AGENT_UNIT"; do + if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then + printf '%s: %s\n' "$unit" "$(systemctl is-active "$unit" 2>/dev/null || true)" + else + printf '%s: not-installed\n' "$unit" + fi + done + if [[ -f "$ETC_DIR/control-ca.crt" ]] && command -v openssl >/dev/null; then + printf 'control-ca: %s\n' "$(openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256 | cut -d= -f2)" + fi +} + +run_control_cli() { + local command_name="$1" identifier="$2" + require_root + validate_node_id "$identifier" + [[ -x "$LIB_DIR/control/ochenstarik-smm-control" ]] || fail "Control is not installed." + [[ -f "$ETC_DIR/control.env" ]] || fail "Control environment is missing." + require_command systemd-run + systemd-run --wait --pipe --quiet --collect \ + --uid="$CONTROL_USER" \ + --gid="$CONTROL_USER" \ + -p "EnvironmentFile=$ETC_DIR/control.env" \ + "$LIB_DIR/control/ochenstarik-smm-control" "$command_name" "$identifier" +} + +show_ca_fingerprint() { + [[ -f "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is not installed." + require_command openssl + openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256 +} + +uninstall_agent() { + local purge="${1:-}" + require_root + systemctl disable --now "$AGENT_UNIT" 2>/dev/null || true + rm -f -- "/etc/systemd/system/$AGENT_UNIT" "$ETC_DIR/agent.env" + rm -rf -- "$LIB_DIR/agent" + [[ "$purge" == "--purge" ]] && rm -rf -- "$STATE_DIR/agent" "$ETC_DIR/control-ca.crt" + systemctl daemon-reload + log "Agent removed${purge:+ ($purge)}." +} + +uninstall_control() { + [[ "${1:-}" == "--confirm-destroy-control" ]] || fail "Control removal requires --confirm-destroy-control" + require_root + systemctl disable --now "$CONTROL_UNIT" 2>/dev/null || true + rm -f -- "/etc/systemd/system/$CONTROL_UNIT" "$ETC_DIR/control.env" \ + "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx" "$ETC_DIR/control-ca.crt" \ + "$POLICY_HELPER" "$SUDOERS_FILE" + rm -rf -- "$LIB_DIR/control" "$STATE_DIR/control.db" "$STATE_DIR/control.db-wal" \ + "$STATE_DIR/control.db-shm" "$STATE_DIR/backups" + systemctl daemon-reload + log "Control role and its state were removed." +} + +preflight() { + validate_platform + local command_name + for command_name in openssl sha256sum tar systemctl getent useradd groupadd; do + require_command "$command_name" + done + log "Supported platform: $(. /etc/os-release; printf '%s %s' "$ID" "$VERSION_ID"), $(uname -m)" +} + +main() { + local action="${1:-help}" + shift || true + case "$action" in + help|-h|--help) usage ;; + version|--version) printf '%s %s\n' "$PROGRAM" "$VERSION" ;; + preflight) preflight ;; + install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;; + install-agent) [[ $# -eq 4 ]] || fail "install-agent requires ARCHIVE NODE_ID CONTROL_URL CA_CERT"; install_agent "$@" ;; + update-control) [[ $# -eq 1 ]] || fail "update-control requires ARCHIVE"; update_role control "$1" ;; + update-agent) [[ $# -eq 1 ]] || fail "update-agent requires ARCHIVE"; update_role agent "$1" ;; + rollback) [[ $# -ge 1 && $# -le 2 ]] || fail "rollback requires control|agent [BACKUP_ID]"; rollback_role "$@" ;; + node-token) [[ $# -eq 1 ]] || fail "node-token requires NODE_ID"; run_control_cli token-create "$1" ;; + control-ca-fingerprint) [[ $# -eq 0 ]] || fail "control-ca-fingerprint takes no arguments"; show_ca_fingerprint ;; + status) show_status ;; + uninstall-agent) [[ $# -le 1 ]] || fail "uninstall-agent accepts only [--purge]"; uninstall_agent "${1:-}" ;; + uninstall-control) [[ $# -eq 1 ]] || fail "uninstall-control requires confirmation"; uninstall_control "$1" ;; + *) fail "Unknown action: $action (run with --help)" ;; + esac +} + +main "$@" diff --git a/deploy/ochenstarik-smm-policy-apply b/deploy/ochenstarik-smm-policy-apply new file mode 100755 index 0000000..00a492b --- /dev/null +++ b/deploy/ochenstarik-smm-policy-apply @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; } + +[[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required" +action="${1:-}" +case "$action" in + link-connect) + [[ $# -eq 6 ]] || fail "invalid link-connect argument count" + ;; + link-disconnect) + [[ $# -eq 5 ]] || fail "invalid link-disconnect argument count" + ;; + *) fail "unsupported action" ;; +esac + +node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' +[[ "$2" =~ $node_pattern && "$3" =~ $node_pattern ]] || fail "invalid node id" +[[ "$4" == "tcp" || "$4" == "udp" ]] || fail "invalid protocol" +[[ "$5" =~ ^[0-9]+$ ]] && (( 10#$5 >= 1 && 10#$5 <= 65535 )) || fail "invalid port" +if [[ "$action" == "link-connect" ]]; then + [[ "$6" =~ ^[0-9]+$ ]] && (( 10#$6 >= 1 && 10#$6 <= 10080 )) || fail "invalid TTL" +fi + +fail "WireGuard/nftables data-plane configuration is not installed yet" diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md new file mode 100644 index 0000000..f84795b --- /dev/null +++ b/docs/linux-bootstrap.md @@ -0,0 +1,83 @@ +# Linux bootstrap: первая тестовая версия + +Собственный bootstrap Server Monitor Manager устанавливает Control или Agent из локального release-архива. Архив принимается только вместе с файлом `.sha256`, проверяется до распаковки и может содержать лишь каталоги `agent`, `control`, `deploy` и `bootstrap`. + +Текущая версия предназначена для подготовки следующего alpha release. Она устанавливает Control/Agent и mTLS enrollment, но пока не настраивает WireGuard data plane. Ограниченный policy helper намеренно отклоняет Link mutations до реализации следующего этапа. + +## Поддерживаемые системы + +- Ubuntu Server 22.04/24.04; +- Debian 12/13; +- `amd64` и `arm64`; +- systemd. + +## Установка Control + +Скачайте из одного release: + +- `ochenstarik-server-monitor-manager.sh` и его `.sha256`; +- `server-monitor-manager-linux-x64.tar.gz` или `server-monitor-manager-linux-arm64.tar.gz`; +- соответствующий `.tar.gz.sha256`. + +Сначала проверьте bootstrap, затем установите Control. `PUBLIC_HOST` должен совпадать с DNS-именем или IP, по которому Agents обращаются к Hub: + +```bash +sha256sum -c ochenstarik-server-monitor-manager.sh.sha256 +chmod 700 ochenstarik-server-monitor-manager.sh +sudo ./ochenstarik-server-monitor-manager.sh preflight +sudo ./ochenstarik-server-monitor-manager.sh install-control \ + ./server-monitor-manager-linux-x64.tar.gz \ + hub.example.com \ + 7443 +``` + +Bootstrap создаёт собственный Control CA и HTTPS certificate, системного пользователя, root-only configuration, SQLite directory и hardened systemd unit. В конце он показывает SHA-256 fingerprint CA. + +Создайте десятиминутный token для Node: + +```bash +sudo ./ochenstarik-server-monitor-manager.sh node-token home +sudo ./ochenstarik-server-monitor-manager.sh control-ca-fingerprint +``` + +Скопируйте на Node только публичный файл `/etc/ochenstarik-server-monitor-manager/control-ca.crt`. Приватный CA PFX остаётся на Hub. + +## Установка Agent + +`CONTROL_URL` обязан использовать то же DNS-имя/IP, которое было передано при установке Control. Token вводится скрыто в локальном терминале и не сохраняется в `agent.env`: + +```bash +sudo ./ochenstarik-server-monitor-manager.sh install-agent \ + ./server-monitor-manager-linux-x64.tar.gz \ + home \ + https://hub.example.com:7443 \ + ./control-ca.crt +``` + +Для автоматизированного теста token можно передать только в окружении одного процесса: + +```bash +sudo SMM_ENROLL_TOKEN='one-time-token' \ + ./ochenstarik-server-monitor-manager.sh install-agent \ + ./server-monitor-manager-linux-x64.tar.gz home \ + https://hub.example.com:7443 ./control-ca.crt +``` + +## Жизненный цикл + +```bash +sudo ./ochenstarik-server-monitor-manager.sh status +sudo ./ochenstarik-server-monitor-manager.sh update-control ARCHIVE +sudo ./ochenstarik-server-monitor-manager.sh update-agent ARCHIVE +sudo ./ochenstarik-server-monitor-manager.sh rollback control +sudo ./ochenstarik-server-monitor-manager.sh rollback agent +sudo ./ochenstarik-server-monitor-manager.sh uninstall-agent +sudo ./ochenstarik-server-monitor-manager.sh uninstall-agent --purge +sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control +``` + +Update создаёт root-only backup перед заменой binaries. При неуспешном health state предыдущая версия восстанавливается. Agent uninstall без `--purge` сохраняет state; удаление Control требует явного подтверждения и удаляет его SQLite/CA state. + +## Следующий этап + +До тестирования Mesh необходимо добавить WireGuard Hub/Node setup, выдачу адресов, persistent keepalive и применение nftables Links в policy helper. До этого момента bootstrap подходит для тестирования Control, Agent, enrollment, heartbeat, update и rollback, но не для сетевого объединения серверов. diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh new file mode 100755 index 0000000..28afcae --- /dev/null +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" +helper="$root/deploy/ochenstarik-smm-policy-apply" + +help_output="$(bash "$bootstrap" --help)" +version_output="$(bash "$bootstrap" --version)" + +grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output" +grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output" +grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output" +grep -Fq "node-token NODE_ID" <<<"$help_output" +grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" + +if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then + printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2 + exit 1 +fi + +if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>&1; then + printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2 + exit 1 +fi + +printf '%s\n' "BOOTSTRAP_CONTRACT=PASS" -- 2.45.2 From d3da396c7e387384d4c4b948407beea85c92939d Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:18:22 +0700 Subject: [PATCH 02/31] Record bootstrap foundation progress --- docs/roadmap.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 76d60d7..c0e5227 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,12 +90,15 @@ ## Этап 7 — собственный bootstrap Server Monitor Manager -- [ ] добавить bootstrap source в этот репозиторий; -- [ ] публиковать bootstrap, checksum и signed compatibility manifest в release; -- [ ] поддержать Ubuntu 22.04/24.04 и Debian 12/13, `amd64`/`arm64`; -- [ ] добавить non-interactive install/update/rollback/uninstall; -- [ ] устанавливать Agent, restricted helper и systemd units; -- [ ] локально создавать Node keys/CSR и погашать bootstrap enrollment; +- [x] добавить bootstrap source в этот репозиторий; +- [x] упаковывать bootstrap, checksum и compatibility manifest в release workflow; +- [ ] добавить криптографическую подпись compatibility manifest для production release; +- [x] проверять Ubuntu 22.04/24.04 и Debian 12/13, `amd64`/`arm64`; +- [x] добавить non-interactive install/update/rollback/uninstall; +- [x] устанавливать Control/Agent, restricted helper и systemd units; +- [x] локально создавать Agent key/CSR, выполнять mTLS enrollment и не сохранять token; +- [ ] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; +- [ ] реализовать nftables policy helper вместо временного deny-by-default helper; - [ ] добавить VM CI matrix, повторную установку и reboot; - [ ] добавить локальную emergency recovery command. -- 2.45.2 From d91f2d380c5604118455f59d792d0d43cce45ff7 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:22:11 +0700 Subject: [PATCH 03/31] Add single-code Node enrollment --- deploy/ochenstarik-server-monitor-manager.sh | 91 ++++++++++++++++++++ docs/linux-bootstrap.md | 24 +++--- tests/bootstrap/test-bootstrap-contract.sh | 2 + 3 files changed, 103 insertions(+), 14 deletions(-) diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index d556f3c..d27dd4d 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -35,9 +35,11 @@ Usage: ochenstarik-server-monitor-manager.sh preflight ochenstarik-server-monitor-manager.sh install-control ARCHIVE PUBLIC_HOST [HTTPS_PORT] ochenstarik-server-monitor-manager.sh install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT + ochenstarik-server-monitor-manager.sh install-node ARCHIVE ochenstarik-server-monitor-manager.sh update-control ARCHIVE ochenstarik-server-monitor-manager.sh update-agent ARCHIVE ochenstarik-server-monitor-manager.sh rollback control|agent [BACKUP_ID] + ochenstarik-server-monitor-manager.sh node-code NODE_ID ochenstarik-server-monitor-manager.sh node-token NODE_ID ochenstarik-server-monitor-manager.sh control-ca-fingerprint ochenstarik-server-monitor-manager.sh status @@ -51,6 +53,23 @@ written to agent.env. EOF } +base64url_encode() { + base64 -w 0 | tr '+/' '-_' | tr -d '=' +} + +base64url_decode() { + local value="$1" remainder + [[ "$value" =~ ^[A-Za-z0-9_-]+$ ]] || fail "Enrollment code contains invalid base64url data." + remainder=$(( ${#value} % 4 )) + case "$remainder" in + 0) ;; + 2) value+="==" ;; + 3) value+="=" ;; + *) fail "Enrollment code contains invalid base64url length." ;; + esac + printf '%s' "$value" | tr '_-' '/+' | base64 -d +} + require_root() { [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "This action must run as root (use sudo)." } @@ -264,8 +283,10 @@ Control__BackupDirectory=$STATE_DIR/backups Control__HubHelperPath=$POLICY_HELPER Control__PrivilegeEscalationPath=/usr/bin/sudo EOF + printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url" chown root:"$CONTROL_USER" "$ETC_DIR/control.env" chmod 0640 "$ETC_DIR/control.env" + chmod 0644 "$ETC_DIR/control-public-url" install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER" printf '%s\n' "$CONTROL_USER ALL=(root) NOPASSWD: $POLICY_HELPER *" >"$SUDOERS_FILE" chmod 0440 "$SUDOERS_FILE" @@ -346,6 +367,54 @@ EOF log "Agent $node_id installed and enrolled. Backup: $backup_id" } +read_enrollment_code() { + if [[ -n "${SMM_ENROLL_CODE:-}" ]]; then + ENROLL_CODE="$SMM_ENROLL_CODE" + unset SMM_ENROLL_CODE + return + fi + [[ -t 0 ]] || fail "Set SMM_ENROLL_CODE or run from an interactive local terminal." + read -r -s -p "SMMNODE1 enrollment code: " ENROLL_CODE + printf '\n' + [[ -n "$ENROLL_CODE" ]] || fail "Enrollment code is empty." +} + +confirm_ca_fingerprint() { + local ca_file="$1" answer + log "Control CA fingerprint: $(openssl x509 -in "$ca_file" -noout -fingerprint -sha256 | cut -d= -f2)" + if [[ "${SMM_ACCEPT_CA_FINGERPRINT:-}" == "1" ]]; then + return + fi + [[ -t 0 ]] || fail "Set SMM_ACCEPT_CA_FINGERPRINT=1 only after verifying the fingerprint out of band." + read -r -p "Type 'yes' after comparing this fingerprint with the Hub: " answer + [[ "$answer" == "yes" ]] || fail "Control CA fingerprint was not confirmed." +} + +install_node_from_code() { + local archive="$1" prefix control_part ca_part node_part token_part extra + local control_url node_id token ca_file + require_root + require_command base64 + require_command openssl + read_enrollment_code + IFS='.' read -r prefix control_part ca_part node_part token_part extra <<<"$ENROLL_CODE" + ENROLL_CODE="" + [[ "$prefix" == "SMMNODE1" && -n "$control_part" && -n "$ca_part" \ + && -n "$node_part" && -n "$token_part" && -z "${extra:-}" ]] \ + || fail "Invalid SMMNODE1 enrollment code." + control_url="$(base64url_decode "$control_part")" + node_id="$(base64url_decode "$node_part")" + token="$(base64url_decode "$token_part")" + ca_file="$(mktemp -t smm-control-ca.XXXXXXXX.crt)" + chmod 0600 "$ca_file" + base64url_decode "$ca_part" >"$ca_file" + confirm_ca_fingerprint "$ca_file" + SMM_ENROLL_TOKEN="$token" + token="" + install_agent "$archive" "$node_id" "$control_url" "$ca_file" + rm -f -- "$ca_file" +} + update_role() { local role="$1" archive="$2" binary unit user backup_id require_root @@ -423,6 +492,26 @@ run_control_cli() { "$LIB_DIR/control/ochenstarik-smm-control" "$command_name" "$identifier" } +create_node_code() { + local node_id="$1" token control_url ca_pem + require_root + validate_node_id "$node_id" + [[ -r "$ETC_DIR/control-public-url" ]] || fail "Control public URL is missing; reinstall Control with PUBLIC_HOST." + [[ -r "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is missing." + require_command base64 + control_url="$(tr -d '\r\n' <"$ETC_DIR/control-public-url")" + validate_control_url "$control_url" + token="$(run_control_cli token-create "$node_id")" + [[ -n "$token" && "$token" != *$'\n'* ]] || fail "Control returned an invalid enrollment token." + ca_pem="$(cat "$ETC_DIR/control-ca.crt")" + printf 'SMMNODE1.%s.%s.%s.%s\n' \ + "$(printf '%s' "$control_url" | base64url_encode)" \ + "$(printf '%s' "$ca_pem" | base64url_encode)" \ + "$(printf '%s' "$node_id" | base64url_encode)" \ + "$(printf '%s' "$token" | base64url_encode)" + token="" +} + show_ca_fingerprint() { [[ -f "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is not installed." require_command openssl @@ -471,9 +560,11 @@ main() { preflight) preflight ;; install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;; install-agent) [[ $# -eq 4 ]] || fail "install-agent requires ARCHIVE NODE_ID CONTROL_URL CA_CERT"; install_agent "$@" ;; + install-node) [[ $# -eq 1 ]] || fail "install-node requires ARCHIVE"; install_node_from_code "$1" ;; update-control) [[ $# -eq 1 ]] || fail "update-control requires ARCHIVE"; update_role control "$1" ;; update-agent) [[ $# -eq 1 ]] || fail "update-agent requires ARCHIVE"; update_role agent "$1" ;; rollback) [[ $# -ge 1 && $# -le 2 ]] || fail "rollback requires control|agent [BACKUP_ID]"; rollback_role "$@" ;; + node-code) [[ $# -eq 1 ]] || fail "node-code requires NODE_ID"; create_node_code "$1" ;; node-token) [[ $# -eq 1 ]] || fail "node-token requires NODE_ID"; run_control_cli token-create "$1" ;; control-ca-fingerprint) [[ $# -eq 0 ]] || fail "control-ca-fingerprint takes no arguments"; show_ca_fingerprint ;; status) show_status ;; diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md index f84795b..0bf3d52 100644 --- a/docs/linux-bootstrap.md +++ b/docs/linux-bootstrap.md @@ -33,34 +33,30 @@ sudo ./ochenstarik-server-monitor-manager.sh install-control \ Bootstrap создаёт собственный Control CA и HTTPS certificate, системного пользователя, root-only configuration, SQLite directory и hardened systemd unit. В конце он показывает SHA-256 fingerprint CA. -Создайте десятиминутный token для Node: +Создайте десятиминутный самодостаточный код `SMMNODE1` для Node: ```bash -sudo ./ochenstarik-server-monitor-manager.sh node-token home +sudo ./ochenstarik-server-monitor-manager.sh node-code home sudo ./ochenstarik-server-monitor-manager.sh control-ca-fingerprint ``` -Скопируйте на Node только публичный файл `/etc/ochenstarik-server-monitor-manager/control-ca.crt`. Приватный CA PFX остаётся на Hub. +Код содержит URL Control, Node ID, одноразовый token и только публичный CA certificate. Приватный CA PFX остаётся на Hub. ## Установка Agent -`CONTROL_URL` обязан использовать то же DNS-имя/IP, которое было передано при установке Control. Token вводится скрыто в локальном терминале и не сохраняется в `agent.env`: +Вставьте код `SMMNODE1` в скрытый prompt. Bootstrap покажет fingerprint CA и потребует сравнить его с Hub до создания локального Agent key и CSR. Token не сохраняется в `agent.env`: ```bash -sudo ./ochenstarik-server-monitor-manager.sh install-agent \ - ./server-monitor-manager-linux-x64.tar.gz \ - home \ - https://hub.example.com:7443 \ - ./control-ca.crt +sudo ./ochenstarik-server-monitor-manager.sh install-node \ + ./server-monitor-manager-linux-x64.tar.gz ``` -Для автоматизированного теста token можно передать только в окружении одного процесса: +Для автоматизированного теста код и явное принятие уже сверенного fingerprint можно передать только в окружении одного процесса: ```bash -sudo SMM_ENROLL_TOKEN='one-time-token' \ - ./ochenstarik-server-monitor-manager.sh install-agent \ - ./server-monitor-manager-linux-x64.tar.gz home \ - https://hub.example.com:7443 ./control-ca.crt +sudo SMM_ENROLL_CODE='SMMNODE1....' SMM_ACCEPT_CA_FINGERPRINT=1 \ + ./ochenstarik-server-monitor-manager.sh install-node \ + ./server-monitor-manager-linux-x64.tar.gz ``` ## Жизненный цикл diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 28afcae..42f5ec4 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -11,7 +11,9 @@ version_output="$(bash "$bootstrap" --version)" grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output" grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output" +grep -Fq "install-node ARCHIVE" <<<"$help_output" grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output" +grep -Fq "node-code NODE_ID" <<<"$help_output" grep -Fq "node-token NODE_ID" <<<"$help_output" grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" -- 2.45.2 From db995d26a09b23c87f7a255bcd88662dfedda867 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:25:06 +0700 Subject: [PATCH 04/31] Validate bootstrap release payloads --- deploy/ochenstarik-server-monitor-manager.sh | 14 ++++++++++++++ docs/linux-bootstrap.md | 2 ++ tests/bootstrap/test-bootstrap-contract.sh | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index d27dd4d..3e63eb0 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -33,6 +33,7 @@ Server Monitor Manager Linux bootstrap Usage: ochenstarik-server-monitor-manager.sh preflight + ochenstarik-server-monitor-manager.sh verify-release ARCHIVE ochenstarik-server-monitor-manager.sh install-control ARCHIVE PUBLIC_HOST [HTTPS_PORT] ochenstarik-server-monitor-manager.sh install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT ochenstarik-server-monitor-manager.sh install-node ARCHIVE @@ -143,6 +144,18 @@ extract_archive() { [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." } +verify_release_payload() { + local archive="$1" + require_command sha256sum + require_command tar + extract_archive "$archive" + [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." + [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing." + [[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing." + log "Release archive and checksum are valid." +} + ensure_system_user() { local user="$1" if ! getent group "$user" >/dev/null; then @@ -558,6 +571,7 @@ main() { help|-h|--help) usage ;; version|--version) printf '%s %s\n' "$PROGRAM" "$VERSION" ;; preflight) preflight ;; + verify-release) [[ $# -eq 1 ]] || fail "verify-release requires ARCHIVE"; verify_release_payload "$1" ;; install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;; install-agent) [[ $# -eq 4 ]] || fail "install-agent requires ARCHIVE NODE_ID CONTROL_URL CA_CERT"; install_agent "$@" ;; install-node) [[ $# -eq 1 ]] || fail "install-node requires ARCHIVE"; install_node_from_code "$1" ;; diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md index 0bf3d52..8b2c94c 100644 --- a/docs/linux-bootstrap.md +++ b/docs/linux-bootstrap.md @@ -25,6 +25,8 @@ sha256sum -c ochenstarik-server-monitor-manager.sh.sha256 chmod 700 ochenstarik-server-monitor-manager.sh sudo ./ochenstarik-server-monitor-manager.sh preflight +sudo ./ochenstarik-server-monitor-manager.sh verify-release \ + ./server-monitor-manager-linux-x64.tar.gz sudo ./ochenstarik-server-monitor-manager.sh install-control \ ./server-monitor-manager-linux-x64.tar.gz \ hub.example.com \ diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 42f5ec4..004f7ff 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -14,6 +14,7 @@ grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output" grep -Fq "install-node ARCHIVE" <<<"$help_output" grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output" grep -Fq "node-code NODE_ID" <<<"$help_output" +grep -Fq "verify-release ARCHIVE" <<<"$help_output" grep -Fq "node-token NODE_ID" <<<"$help_output" grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" @@ -27,4 +28,23 @@ if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 exit 1 fi +fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)" +trap 'rm -rf -- "$fixture"' EXIT +mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/deploy" "$fixture/payload/bootstrap" +install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent" +install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" +install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" +install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" +install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" +install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh" +tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control deploy bootstrap +sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256" +bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null + +printf '%064d %s\n' 0 release.tar.gz >"$fixture/release.tar.gz.sha256" +if bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null 2>&1; then + printf '%s\n' "corrupt release checksum unexpectedly succeeded" >&2 + exit 1 +fi + printf '%s\n' "BOOTSTRAP_CONTRACT=PASS" -- 2.45.2 From 661919be39f54ab7ab861959496dc3504dcf4d5f Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:45:53 +0700 Subject: [PATCH 05/31] Add managed WireGuard mesh bootstrap --- .github/workflows/linux-release.yml | 1 + deploy/ochenstarik-server-monitor-manager.sh | 310 ++++++++++++++++++- deploy/ochenstarik-smm-firewall.service | 15 + deploy/ochenstarik-smm-policy-apply | 118 +++++-- docs/linux-bootstrap.md | 71 +++-- docs/roadmap.md | 4 +- tests/bootstrap/test-bootstrap-contract.sh | 16 + 7 files changed, 474 insertions(+), 61 deletions(-) create mode 100644 deploy/ochenstarik-smm-firewall.service diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index 3b81bd2..5ab5706 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -85,6 +85,7 @@ jobs: install -d out/deploy out/bootstrap install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz" diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 3e63eb0..5f0f307 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -14,8 +14,14 @@ readonly CONTROL_UNIT="ochenstarik-smm-control.service" readonly AGENT_UNIT="ochenstarik-smm-agent.service" readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply" readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control" +readonly MESH_DIR="${STATE_DIR}/mesh" +readonly WG_DIR="${ETC_DIR}/wireguard" +readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service" +readonly MESH_NETWORK="10.77.0.0/24" +readonly HUB_MESH_ADDRESS="10.77.0.1/24" TEMP_DIR="" +MESH_PEER_CODE="" log() { printf '%s\n' "[$PROGRAM] $*"; } fail() { printf '%s\n' "[$PROGRAM] ERROR: $*" >&2; exit 1; } @@ -37,6 +43,9 @@ Usage: ochenstarik-server-monitor-manager.sh install-control ARCHIVE PUBLIC_HOST [HTTPS_PORT] ochenstarik-server-monitor-manager.sh install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT ochenstarik-server-monitor-manager.sh install-node ARCHIVE + ochenstarik-server-monitor-manager.sh mesh-init PUBLIC_ENDPOINT [WG_PORT] + ochenstarik-server-monitor-manager.sh peer-add SMMPEER1_CODE + ochenstarik-server-monitor-manager.sh mesh-status ochenstarik-server-monitor-manager.sh update-control ARCHIVE ochenstarik-server-monitor-manager.sh update-agent ARCHIVE ochenstarik-server-monitor-manager.sh rollback control|agent [BACKUP_ID] @@ -142,6 +151,7 @@ extract_archive() { tar -xzf "$archive" -C "$TEMP_DIR" --no-same-owner --no-same-permissions [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." + [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive." } verify_release_payload() { @@ -152,6 +162,7 @@ verify_release_payload() { [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing." + [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing." [[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing." log "Release archive and checksum are valid." } @@ -166,6 +177,19 @@ ensure_system_user() { fi } +ensure_mesh_packages() { + local missing=0 command_name + for command_name in wg wg-quick nft ip; do + command -v "$command_name" >/dev/null 2>&1 || missing=1 + done + (( missing == 0 )) && return + require_command apt-get + log "Installing WireGuard/nftables dependencies." + DEBIAN_FRONTEND=noninteractive apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wireguard-tools nftables iproute2 +} + install_tree_atomic() { local source="$1" destination="$2" owner="$3" staging [[ -d "$source" ]] || fail "Release payload is missing: $source" @@ -227,6 +251,163 @@ install_unit() { systemctl daemon-reload } +write_mesh_firewall() { + cat >"$ETC_DIR/mesh.nft" <<'EOF' +table inet ochenstarik_smm { + chain links { + ct state established,related accept + counter drop + } + + chain mesh_forward { + type filter hook forward priority filter; policy accept; + iifname "smm0" oifname "smm0" jump links + } + } +EOF + chmod 0644 "$ETC_DIR/mesh.nft" + if ! nft list table inet ochenstarik_smm >/dev/null 2>&1; then + nft --check -f "$ETC_DIR/mesh.nft" + fi +} + +read_mesh_value() { + local key="$1" + [[ -r "$ETC_DIR/mesh.env" ]] || fail "Mesh Hub is not initialized." + awk -F '=' -v key="$key" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "$ETC_DIR/mesh.env" +} + +render_hub_wireguard_config() { + local private_key endpoint port node_id address public_key status + private_key="$(cat "$WG_DIR/hub.key")" + endpoint="$(read_mesh_value HUB_ENDPOINT)" + port="${endpoint##*:}" + cat >"/etc/wireguard/smm0.conf" <>"/etc/wireguard/smm0.conf" <"$WG_DIR/hub.key" + fi + hub_private="$(cat "$WG_DIR/hub.key")" + hub_public="$(printf '%s' "$hub_private" | wg pubkey)" + printf '%s\n' "$hub_public" >"$WG_DIR/hub.pub" + chmod 0600 "$WG_DIR/hub.key" + chmod 0644 "$WG_DIR/hub.pub" + cat >"$ETC_DIR/mesh.env" <"/etc/sysctl.d/90-ochenstarik-smm-mesh.conf" + sysctl --system >/dev/null + write_mesh_firewall + [[ -f "$LIB_DIR/control/ochenstarik-smm-control" ]] \ + || log "Warning: Control is not installed yet; mesh peer codes require Control enrollment." + if [[ -f "${TEMP_DIR:-}/deploy/$FIREWALL_UNIT" ]]; then + install_unit "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$FIREWALL_UNIT" + elif [[ -f "$LIB_DIR/bootstrap/$FIREWALL_UNIT" ]]; then + install_unit "$LIB_DIR/bootstrap/$FIREWALL_UNIT" "$FIREWALL_UNIT" + else + fail "Mesh firewall systemd unit is unavailable; reinstall Control from the current release." + fi + systemctl enable "$FIREWALL_UNIT" + systemctl restart "$FIREWALL_UNIT" + render_hub_wireguard_config + systemctl enable wg-quick@smm0.service + systemctl restart wg-quick@smm0.service + log "Mesh Hub initialized at $public_endpoint:$port with $MESH_NETWORK." + log "WireGuard public key: $hub_public" +} + +reserve_node_address() { + local node_id="$1" existing host address + install -d -m 0700 "$MESH_DIR" + touch "$MESH_DIR/nodes.tsv" + chmod 0600 "$MESH_DIR/nodes.tsv" + existing="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$MESH_DIR/nodes.tsv")" + if [[ -n "$existing" ]]; then + printf '%s\n' "$existing" + return + fi + for host in $(seq 2 254); do + address="10.77.0.$host" + if ! awk -F '\t' -v address="$address" '$2 == address { found=1 } END { exit found ? 0 : 1 }' "$MESH_DIR/nodes.tsv"; then + printf '%s\t%s\t-\treserved\n' "$node_id" "$address" >>"$MESH_DIR/nodes.tsv" + printf '%s\n' "$address" + return + fi + done + fail "Mesh address pool is exhausted." +} + +configure_node_wireguard() { + local node_id="$1" node_address="$2" hub_endpoint="$3" hub_public_key="$4" node_private node_public + ensure_mesh_packages + install -d -m 0700 "$WG_DIR" /etc/wireguard + if [[ ! -f "$WG_DIR/node.key" ]]; then + umask 077 + wg genkey >"$WG_DIR/node.key" + fi + node_private="$(cat "$WG_DIR/node.key")" + node_public="$(printf '%s' "$node_private" | wg pubkey)" + printf '%s\n' "$node_public" >"$WG_DIR/node.pub" + cat >"/etc/wireguard/smm0.conf" <"$SUDOERS_FILE" chmod 0440 "$SUDOERS_FILE" visudo -cf "$SUDOERS_FILE" >/dev/null @@ -387,7 +570,7 @@ read_enrollment_code() { return fi [[ -t 0 ]] || fail "Set SMM_ENROLL_CODE or run from an interactive local terminal." - read -r -s -p "SMMNODE1 enrollment code: " ENROLL_CODE + read -r -s -p "SMMNODE enrollment code: " ENROLL_CODE printf '\n' [[ -n "$ENROLL_CODE" ]] || fail "Enrollment code is empty." } @@ -404,20 +587,43 @@ confirm_ca_fingerprint() { } install_node_from_code() { - local archive="$1" prefix control_part ca_part node_part token_part extra - local control_url node_id token ca_file + local archive="$1" prefix control_part ca_part node_part token_part + local endpoint_part hub_key_part address_part network_part extra + local control_url node_id token ca_file hub_endpoint hub_public_key node_address mesh_network require_root require_command base64 require_command openssl read_enrollment_code - IFS='.' read -r prefix control_part ca_part node_part token_part extra <<<"$ENROLL_CODE" + IFS='.' read -r prefix control_part ca_part node_part token_part endpoint_part \ + hub_key_part address_part network_part extra <<<"$ENROLL_CODE" ENROLL_CODE="" - [[ "$prefix" == "SMMNODE1" && -n "$control_part" && -n "$ca_part" \ - && -n "$node_part" && -n "$token_part" && -z "${extra:-}" ]] \ - || fail "Invalid SMMNODE1 enrollment code." + [[ "$prefix" == "SMMNODE1" || "$prefix" == "SMMNODE2" ]] \ + || fail "Unsupported SMMNODE enrollment code version." + [[ -n "$control_part" && -n "$ca_part" && -n "$node_part" && -n "$token_part" \ + && -z "${extra:-}" ]] || fail "Invalid SMMNODE enrollment code." + if [[ "$prefix" == "SMMNODE1" ]]; then + [[ -z "${endpoint_part:-}${hub_key_part:-}${address_part:-}${network_part:-}" ]] \ + || fail "Invalid SMMNODE1 enrollment code." + else + [[ -n "${endpoint_part:-}" && -n "${hub_key_part:-}" \ + && -n "${address_part:-}" && -n "${network_part:-}" ]] \ + || fail "Invalid SMMNODE2 mesh enrollment code." + fi control_url="$(base64url_decode "$control_part")" node_id="$(base64url_decode "$node_part")" token="$(base64url_decode "$token_part")" + if [[ "$prefix" == "SMMNODE2" ]]; then + hub_endpoint="$(base64url_decode "$endpoint_part")" + hub_public_key="$(base64url_decode "$hub_key_part")" + node_address="$(base64url_decode "$address_part")" + mesh_network="$(base64url_decode "$network_part")" + [[ "$hub_endpoint" =~ ^[A-Za-z0-9.-]+:[0-9]{1,5}$ ]] || fail "Invalid Hub WireGuard endpoint." + validate_port "${hub_endpoint##*:}" + [[ "$hub_public_key" =~ ^[A-Za-z0-9+/]{43}=$ ]] || fail "Invalid Hub WireGuard public key." + [[ "$node_address" =~ ^10\.77\.0\.([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])$ ]] \ + || fail "Invalid reserved mesh address." + [[ "$mesh_network" == "$MESH_NETWORK" ]] || fail "Unsupported mesh network." + fi ca_file="$(mktemp -t smm-control-ca.XXXXXXXX.crt)" chmod 0600 "$ca_file" base64url_decode "$ca_part" >"$ca_file" @@ -426,6 +632,12 @@ install_node_from_code() { token="" install_agent "$archive" "$node_id" "$control_url" "$ca_file" rm -f -- "$ca_file" + if [[ "$prefix" == "SMMNODE2" ]]; then + configure_node_wireguard "$node_id" "$node_address" "$hub_endpoint" "$hub_public_key" + log "Node mesh configured. Copy this public peer code back to the Hub:" + printf '%s\n' "$MESH_PEER_CODE" + MESH_PEER_CODE="" + fi } update_role() { @@ -506,7 +718,7 @@ run_control_cli() { } create_node_code() { - local node_id="$1" token control_url ca_pem + local node_id="$1" token control_url ca_pem node_address hub_endpoint hub_public_key mesh_network require_root validate_node_id "$node_id" [[ -r "$ETC_DIR/control-public-url" ]] || fail "Control public URL is missing; reinstall Control with PUBLIC_HOST." @@ -517,14 +729,83 @@ create_node_code() { token="$(run_control_cli token-create "$node_id")" [[ -n "$token" && "$token" != *$'\n'* ]] || fail "Control returned an invalid enrollment token." ca_pem="$(cat "$ETC_DIR/control-ca.crt")" - printf 'SMMNODE1.%s.%s.%s.%s\n' \ - "$(printf '%s' "$control_url" | base64url_encode)" \ - "$(printf '%s' "$ca_pem" | base64url_encode)" \ - "$(printf '%s' "$node_id" | base64url_encode)" \ - "$(printf '%s' "$token" | base64url_encode)" + if [[ -r "$ETC_DIR/mesh.env" && -r "$WG_DIR/hub.pub" ]]; then + node_address="$(reserve_node_address "$node_id")" + hub_endpoint="$(read_mesh_value HUB_ENDPOINT)" + hub_public_key="$(read_mesh_value HUB_PUBLIC_KEY)" + mesh_network="$(read_mesh_value MESH_NETWORK)" + printf 'SMMNODE2.%s.%s.%s.%s.%s.%s.%s.%s\n' \ + "$(printf '%s' "$control_url" | base64url_encode)" \ + "$(printf '%s' "$ca_pem" | base64url_encode)" \ + "$(printf '%s' "$node_id" | base64url_encode)" \ + "$(printf '%s' "$token" | base64url_encode)" \ + "$(printf '%s' "$hub_endpoint" | base64url_encode)" \ + "$(printf '%s' "$hub_public_key" | base64url_encode)" \ + "$(printf '%s' "$node_address" | base64url_encode)" \ + "$(printf '%s' "$mesh_network" | base64url_encode)" + else + printf 'SMMNODE1.%s.%s.%s.%s\n' \ + "$(printf '%s' "$control_url" | base64url_encode)" \ + "$(printf '%s' "$ca_pem" | base64url_encode)" \ + "$(printf '%s' "$node_id" | base64url_encode)" \ + "$(printf '%s' "$token" | base64url_encode)" + fi token="" } +add_mesh_peer() { + local code="$1" prefix node_part address_part key_part extra + local node_id address public_key current tmp + require_root + require_command base64 + require_command wg + [[ -r "$ETC_DIR/mesh.env" && -r "$MESH_DIR/nodes.tsv" ]] || fail "Mesh Hub is not initialized." + IFS='.' read -r prefix node_part address_part key_part extra <<<"$code" + [[ "$prefix" == "SMMPEER1" && -n "$node_part" && -n "$address_part" \ + && -n "$key_part" && -z "${extra:-}" ]] || fail "Invalid SMMPEER1 code." + node_id="$(base64url_decode "$node_part")" + address="$(base64url_decode "$address_part")" + public_key="$(base64url_decode "$key_part")" + validate_node_id "$node_id" + [[ "$address" =~ ^10\.77\.0\.([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])$ ]] \ + || fail "Invalid peer mesh address." + [[ "$public_key" =~ ^[A-Za-z0-9+/]{43}=$ ]] || fail "Invalid peer WireGuard public key." + current="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$MESH_DIR/nodes.tsv")" + [[ "$current" == "$address" ]] || fail "Peer address does not match the Hub reservation." + if awk -F '\t' -v address="$address" -v node="$node_id" '$2 == address && $1 != node { found=1 } END { exit found ? 0 : 1 }' "$MESH_DIR/nodes.tsv"; then + fail "Peer mesh address is already assigned." + fi + tmp="$(mktemp -p "$MESH_DIR" nodes.tsv.XXXXXXXX)" + awk -F '\t' -v OFS='\t' -v node="$node_id" -v address="$address" -v key="$public_key" \ + '$1 == node { print node, address, key, "active"; found=1; next } { print } END { if (!found) exit 1 }' \ + "$MESH_DIR/nodes.tsv" >"$tmp" || { rm -f -- "$tmp"; fail "Peer reservation is missing."; } + chmod 0600 "$tmp" + mv -- "$tmp" "$MESH_DIR/nodes.tsv" + render_hub_wireguard_config + systemctl restart wg-quick@smm0.service + systemctl is-active --quiet wg-quick@smm0.service || fail "WireGuard failed after peer registration." + log "Mesh peer $node_id activated at $address." +} + +show_mesh_status() { + require_root + [[ -r "$ETC_DIR/mesh.env" ]] || fail "Mesh Hub is not initialized." + printf 'endpoint: %s\n' "$(read_mesh_value HUB_ENDPOINT)" + printf 'network: %s\n' "$(read_mesh_value MESH_NETWORK)" + if [[ -r "$MESH_DIR/nodes.tsv" ]]; then + printf '%-24s %-15s %-10s %s\n' NODE ADDRESS STATUS HANDSHAKE + while IFS=$'\t' read -r node_id address public_key status; do + [[ -n "$node_id" ]] || continue + local handshake="-" + if [[ "$status" == "active" ]]; then + handshake="$(wg show smm0 latest-handshakes 2>/dev/null | awk -v key="$public_key" '$1 == key { print $2; exit }')" + [[ -n "$handshake" && "$handshake" != "0" ]] || handshake="never" + fi + printf '%-24s %-15s %-10s %s\n' "$node_id" "$address" "$status" "$handshake" + done <"$MESH_DIR/nodes.tsv" + fi +} + show_ca_fingerprint() { [[ -f "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is not installed." require_command openssl @@ -575,6 +856,9 @@ main() { install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;; install-agent) [[ $# -eq 4 ]] || fail "install-agent requires ARCHIVE NODE_ID CONTROL_URL CA_CERT"; install_agent "$@" ;; install-node) [[ $# -eq 1 ]] || fail "install-node requires ARCHIVE"; install_node_from_code "$1" ;; + mesh-init) [[ $# -ge 1 && $# -le 2 ]] || fail "mesh-init requires PUBLIC_ENDPOINT [WG_PORT]"; mesh_init "$@" ;; + peer-add) [[ $# -eq 1 ]] || fail "peer-add requires SMMPEER1_CODE"; add_mesh_peer "$1" ;; + mesh-status) [[ $# -eq 0 ]] || fail "mesh-status takes no arguments"; show_mesh_status ;; update-control) [[ $# -eq 1 ]] || fail "update-control requires ARCHIVE"; update_role control "$1" ;; update-agent) [[ $# -eq 1 ]] || fail "update-agent requires ARCHIVE"; update_role agent "$1" ;; rollback) [[ $# -ge 1 && $# -le 2 ]] || fail "rollback requires control|agent [BACKUP_ID]"; rollback_role "$@" ;; diff --git a/deploy/ochenstarik-smm-firewall.service b/deploy/ochenstarik-smm-firewall.service new file mode 100644 index 0000000..f2d19e4 --- /dev/null +++ b/deploy/ochenstarik-smm-firewall.service @@ -0,0 +1,15 @@ +[Unit] +Description=Server Monitor Manager Mesh firewall +After=network-pre.target +Before=wg-quick@smm0.service network-online.target +Wants=network-pre.target + +[Service] +Type=oneshot +ExecStartPre=-/usr/sbin/nft delete table inet ochenstarik_smm +ExecStart=/usr/sbin/nft -f /etc/ochenstarik-server-monitor-manager/mesh.nft +ExecReload=/usr/sbin/nft -f /etc/ochenstarik-server-monitor-manager/mesh.nft +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/ochenstarik-smm-policy-apply b/deploy/ochenstarik-smm-policy-apply index 00a492b..040993e 100755 --- a/deploy/ochenstarik-smm-policy-apply +++ b/deploy/ochenstarik-smm-policy-apply @@ -2,26 +2,106 @@ set -Eeuo pipefail IFS=$'\n\t' +readonly TABLE_FAMILY="inet" +readonly TABLE_NAME="ochenstarik_smm" +readonly CHAIN_NAME="links" + fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; } -[[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required" -action="${1:-}" -case "$action" in - link-connect) - [[ $# -eq 6 ]] || fail "invalid link-connect argument count" - ;; - link-disconnect) - [[ $# -eq 5 ]] || fail "invalid link-disconnect argument count" - ;; - *) fail "unsupported action" ;; -esac - -node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' -[[ "$2" =~ $node_pattern && "$3" =~ $node_pattern ]] || fail "invalid node id" -[[ "$4" == "tcp" || "$4" == "udp" ]] || fail "invalid protocol" -[[ "$5" =~ ^[0-9]+$ ]] && (( 10#$5 >= 1 && 10#$5 <= 65535 )) || fail "invalid port" -if [[ "$action" == "link-connect" ]]; then - [[ "$6" =~ ^[0-9]+$ ]] && (( 10#$6 >= 1 && 10#$6 <= 10080 )) || fail "invalid TTL" +testing="${SMM_POLICY_TESTING:-0}" +if [[ "$testing" != "1" ]]; then + [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required" + readonly STATE_FILE="/var/lib/ochenstarik-server-monitor-manager/mesh/nodes.tsv" +else + readonly STATE_FILE="${SMM_POLICY_STATE_FILE:?SMM_POLICY_STATE_FILE is required in testing mode}" fi -fail "WireGuard/nftables data-plane configuration is not installed yet" +node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' +ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$' + +validate_node_id() { + [[ "$1" =~ $node_pattern ]] || fail "invalid node id" +} + +lookup_node_ip() { + local node_id="$1" ip + [[ -r "$STATE_FILE" ]] || fail "mesh node state is unavailable" + ip="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$STATE_FILE")" + [[ "$ip" =~ $ipv4_pattern ]] || fail "node has no valid mesh address: $node_id" + printf '%s\n' "$ip" +} + +validate_rule() { + local action="$1" + case "$action" in + link-connect) [[ $# -eq 6 ]] || fail "invalid link-connect argument count" ;; + link-disconnect) [[ $# -eq 5 ]] || fail "invalid link-disconnect argument count" ;; + *) fail "unsupported action" ;; + esac + validate_node_id "$2" + validate_node_id "$3" + [[ "$2" != "$3" ]] || fail "source and destination must differ" + [[ "$4" == "tcp" || "$4" == "udp" ]] || fail "invalid protocol" + [[ "$5" =~ ^[0-9]+$ ]] && (( 10#$5 >= 1 && 10#$5 <= 65535 )) || fail "invalid port" + if [[ "$action" == "link-connect" ]]; then + [[ "$6" =~ ^[0-9]+$ ]] && (( 10#$6 >= 0 && 10#$6 <= 525600 )) || fail "invalid TTL" + fi +} + +run_nft() { + if [[ "$testing" == "1" ]]; then + printf 'nft' + printf ' %q' "$@" + printf '\n' + else + /usr/sbin/nft "$@" + fi +} + +rule_exists() { + local comment="$1" + if [[ "$testing" == "1" ]]; then + return 1 + fi + /usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \ + | grep -Fq "comment \"$comment\"" +} + +connect_rule() { + local source_id="$1" target_id="$2" protocol="$3" port="$4" + local source_ip target_ip comment + source_ip="$(lookup_node_ip "$source_id")" + target_ip="$(lookup_node_ip "$target_id")" + comment="smm:${source_id}:${target_id}:${protocol}:${port}" + rule_exists "$comment" && return 0 + run_nft add rule "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \ + ip saddr "$source_ip" ip daddr "$target_ip" "$protocol" dport "$port" \ + counter accept comment "$comment" +} + +disconnect_rule() { + local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle + # Resolve both identities before touching firewall state. + lookup_node_ip "$source_id" >/dev/null + lookup_node_ip "$target_id" >/dev/null + comment="smm:${source_id}:${target_id}:${protocol}:${port}" + if [[ "$testing" == "1" ]]; then + printf 'nft-delete-comment %q\n' "$comment" + return + fi + while IFS= read -r handle; do + [[ "$handle" =~ ^[0-9]+$ ]] || continue + run_nft delete rule "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" handle "$handle" + done < <( + /usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \ + | grep -F "comment \"$comment\"" \ + | sed -n 's/.* # handle \([0-9][0-9]*\)$/\1/p' + ) +} + +action="${1:-}" +validate_rule "$@" +case "$action" in + link-connect) connect_rule "$2" "$3" "$4" "$5" ;; + link-disconnect) disconnect_rule "$2" "$3" "$4" "$5" ;; +esac diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md index 8b2c94c..9e32cbe 100644 --- a/docs/linux-bootstrap.md +++ b/docs/linux-bootstrap.md @@ -1,25 +1,20 @@ -# Linux bootstrap: первая тестовая версия +# Linux bootstrap и собственная Mesh-сеть -Собственный bootstrap Server Monitor Manager устанавливает Control или Agent из локального release-архива. Архив принимается только вместе с файлом `.sha256`, проверяется до распаковки и может содержать лишь каталоги `agent`, `control`, `deploy` и `bootstrap`. +Server Monitor Manager устанавливает Control (Hub) и Agent (Node) из проверяемого release-архива. Для связи серверов используется собственная WireGuard-сеть `10.77.0.0/24`; Tailscale и другие внешние VPN-сервисы не требуются. -Текущая версия предназначена для подготовки следующего alpha release. Она устанавливает Control/Agent и mTLS enrollment, но пока не настраивает WireGuard data plane. Ограниченный policy helper намеренно отклоняет Link mutations до реализации следующего этапа. +Текущая версия предназначена для alpha-тестирования на Ubuntu Server 22.04/24.04 и Debian 12/13 (`amd64`, `arm64`, systemd). Hub должен иметь публичный IPv4-адрес или DNS-имя и доступный UDP-порт. Node может находиться за NAT без белого IP. -## Поддерживаемые системы +## Файлы релиза -- Ubuntu Server 22.04/24.04; -- Debian 12/13; -- `amd64` и `arm64`; -- systemd. +Скачайте из одного GitHub Release: -## Установка Control - -Скачайте из одного release: - -- `ochenstarik-server-monitor-manager.sh` и его `.sha256`; +- `ochenstarik-server-monitor-manager.sh` и `.sha256`; - `server-monitor-manager-linux-x64.tar.gz` или `server-monitor-manager-linux-arm64.tar.gz`; - соответствующий `.tar.gz.sha256`. -Сначала проверьте bootstrap, затем установите Control. `PUBLIC_HOST` должен совпадать с DNS-именем или IP, по которому Agents обращаются к Hub: +Bootstrap проверяет SHA-256 до распаковки и принимает в архиве только каталоги `agent`, `control`, `deploy` и `bootstrap`. + +## 1. Установка главного сервера (Hub) ```bash sha256sum -c ochenstarik-server-monitor-manager.sh.sha256 @@ -31,40 +26,66 @@ sudo ./ochenstarik-server-monitor-manager.sh install-control \ ./server-monitor-manager-linux-x64.tar.gz \ hub.example.com \ 7443 +sudo ./ochenstarik-server-monitor-manager.sh mesh-init hub.example.com 51820 ``` -Bootstrap создаёт собственный Control CA и HTTPS certificate, системного пользователя, root-only configuration, SQLite directory и hardened systemd unit. В конце он показывает SHA-256 fingerprint CA. +Откройте на Hub и во внешнем firewall/security group: -Создайте десятиминутный самодостаточный код `SMMNODE1` для Node: +- TCP `7443` для Control HTTPS; +- UDP `51820` для WireGuard. + +`install-control` создаёт локальный Control CA и HTTPS-сертификат. Приватный ключ CA не включается в коды подключения и остаётся на Hub. `mesh-init` устанавливает `wireguard-tools`, `nftables` и `iproute2`, создаёт ключ Hub, интерфейс `smm0`, включает IPv4 forwarding и межсерверный firewall с запретом по умолчанию. + +## 2. Выпуск кода для Node ```bash sudo ./ochenstarik-server-monitor-manager.sh node-code home sudo ./ochenstarik-server-monitor-manager.sh control-ca-fingerprint ``` -Код содержит URL Control, Node ID, одноразовый token и только публичный CA certificate. Приватный CA PFX остаётся на Hub. +После `mesh-init` команда выдаёт одноразовый код `SMMNODE2`. Он содержит Control URL, публичный сертификат CA, Node ID, десятиминутный enrollment token, endpoint и публичный ключ Hub, а также зарезервированный Mesh-адрес. Обращайтесь с кодом как с временным секретом. -## Установка Agent +## 3. Установка вторичного сервера (Node) -Вставьте код `SMMNODE1` в скрытый prompt. Bootstrap покажет fingerprint CA и потребует сравнить его с Hub до создания локального Agent key и CSR. Token не сохраняется в `agent.env`: +Node может быть за NAT. Ему нужен исходящий доступ к TCP-порту Control и UDP-порту WireGuard на Hub. ```bash sudo ./ochenstarik-server-monitor-manager.sh install-node \ ./server-monitor-manager-linux-x64.tar.gz ``` -Для автоматизированного теста код и явное принятие уже сверенного fingerprint можно передать только в окружении одного процесса: +Вставьте `SMMNODE2` в скрытый prompt. Сверьте показанный SHA-256 fingerprint CA с Hub и введите `yes`. После mTLS enrollment установщик создаст локальный приватный ключ WireGuard, запустит `smm0` с `PersistentKeepalive = 25` и выведет публичный код `SMMPEER1`. + +Для автоматизированного стенда допускается передача кода только в окружении процесса после отдельной сверки fingerprint: ```bash -sudo SMM_ENROLL_CODE='SMMNODE1....' SMM_ACCEPT_CA_FINGERPRINT=1 \ +sudo SMM_ENROLL_CODE='SMMNODE2....' SMM_ACCEPT_CA_FINGERPRINT=1 \ ./ochenstarik-server-monitor-manager.sh install-node \ ./server-monitor-manager-linux-x64.tar.gz ``` -## Жизненный цикл +## 4. Активация Node на Hub + +Скопируйте выведенный Node код `SMMPEER1` на Hub: + +```bash +sudo ./ochenstarik-server-monitor-manager.sh peer-add 'SMMPEER1....' +sudo ./ochenstarik-server-monitor-manager.sh mesh-status +``` + +Hub проверяет Node ID и ранее зарезервированный IP, сохраняет публичный ключ peer и перезапускает интерфейс. Приватный ключ Node никогда не покидает Node. + +## Изоляция и управляемые соединения + +Трафик `smm0 -> smm0` по умолчанию блокируется. Control вызывает root-helper только для точных правил `source IP -> target IP`, протокола и порта. Поддерживаются команды helper `link-connect SOURCE TARGET tcp|udp PORT TTL_MINUTES` и `link-disconnect SOURCE TARGET tcp|udp PORT`. Это позволяет вручную подключать AI-агент к выбранному серверу и затем отзывать доступ, не открывая связь между всеми Node. + +В текущем alpha TTL валидируется и хранится Control, а удаление просроченных правил зависит от reconciliation Control. После перезапуска firewall разрешающие правила должны быть повторно применены Control. + +## Обслуживание ```bash sudo ./ochenstarik-server-monitor-manager.sh status +sudo ./ochenstarik-server-monitor-manager.sh mesh-status sudo ./ochenstarik-server-monitor-manager.sh update-control ARCHIVE sudo ./ochenstarik-server-monitor-manager.sh update-agent ARCHIVE sudo ./ochenstarik-server-monitor-manager.sh rollback control @@ -74,8 +95,4 @@ sudo ./ochenstarik-server-monitor-manager.sh uninstall-agent --purge sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control ``` -Update создаёт root-only backup перед заменой binaries. При неуспешном health state предыдущая версия восстанавливается. Agent uninstall без `--purge` сохраняет state; удаление Control требует явного подтверждения и удаляет его SQLite/CA state. - -## Следующий этап - -До тестирования Mesh необходимо добавить WireGuard Hub/Node setup, выдачу адресов, persistent keepalive и применение nftables Links в policy helper. До этого момента bootstrap подходит для тестирования Control, Agent, enrollment, heartbeat, update и rollback, но не для сетевого объединения серверов. +Update создаёт root-only backup перед заменой binaries и автоматически восстанавливает предыдущую версию, если сервис не запускается. Перед alpha-тестом на реальных серверах обязательно сохраните отдельную консольную/SSH-сессию и не закрывайте основной административный доступ firewall-правилами проекта. diff --git a/docs/roadmap.md b/docs/roadmap.md index c0e5227..0665d4a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -97,8 +97,8 @@ - [x] добавить non-interactive install/update/rollback/uninstall; - [x] устанавливать Control/Agent, restricted helper и systemd units; - [x] локально создавать Agent key/CSR, выполнять mTLS enrollment и не сохранять token; -- [ ] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; -- [ ] реализовать nftables policy helper вместо временного deny-by-default helper; +- [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; +- [x] реализовать nftables policy helper вместо временного deny-by-default helper; - [ ] добавить VM CI matrix, повторную установку и reboot; - [ ] добавить локальную emergency recovery command. diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 004f7ff..7b5f5fe 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -12,6 +12,9 @@ version_output="$(bash "$bootstrap" --version)" grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output" grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output" grep -Fq "install-node ARCHIVE" <<<"$help_output" +grep -Fq "mesh-init PUBLIC_ENDPOINT" <<<"$help_output" +grep -Fq "peer-add SMMPEER1_CODE" <<<"$help_output" +grep -Fq "mesh-status" <<<"$help_output" grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output" grep -Fq "node-code NODE_ID" <<<"$help_output" grep -Fq "verify-release ARCHIVE" <<<"$help_output" @@ -28,6 +31,18 @@ if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 exit 1 fi +policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)" +printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state" +printf 'target\t10.77.0.3\tkey-target\tactive\n' >>"$policy_state" +connect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ + bash "$helper" link-connect source target tcp 22 10)" +grep -Fq 'ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22' <<<"$connect_output" +grep -Fq 'smm:source:target:tcp:22' <<<"$connect_output" +disconnect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \ + bash "$helper" link-disconnect source target tcp 22)" +grep -Fq 'smm:source:target:tcp:22' <<<"$disconnect_output" +rm -f -- "$policy_state" + fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)" trap 'rm -rf -- "$fixture"' EXIT mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/deploy" "$fixture/payload/bootstrap" @@ -36,6 +51,7 @@ install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" +install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/" install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh" tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control deploy bootstrap sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256" -- 2.45.2 From 4f472b5852273d01faedd65476709ce944d573b4 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 13:54:09 +0700 Subject: [PATCH 06/31] Exercise repeated systemd installation in CI --- .github/workflows/linux-control-agent.yml | 45 ++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index c860720..17c9c29 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -10,7 +10,7 @@ permissions: jobs: build-and-test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v6 @@ -50,3 +50,46 @@ jobs: - name: Publish agent arm64 run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true + + - name: Build systemd smoke release + run: | + dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj \ + --configuration Release --runtime linux-x64 --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/agent + dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \ + --configuration Release --runtime linux-x64 --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/control + install -d smoke/deploy smoke/bootstrap + install -m 0644 deploy/ochenstarik-smm-control.service smoke/deploy/ + install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/ + install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/ + install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/ + install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/ + tar -C smoke -czf smoke-release.tar.gz agent control deploy bootstrap + sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256 + + - name: Verify repeated systemd installation + run: | + set -Eeuo pipefail + sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ + smoke-release.tar.gz 127.0.0.1 7443 + for _ in {1..30}; do + curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + https://127.0.0.1:7443/healthz && break + sleep 1 + done + curl --fail --silent --show-error \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + https://127.0.0.1:7443/healthz + sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ + smoke-release.tar.gz 127.0.0.1 7443 + sudo systemctl restart ochenstarik-smm-control.service + sudo systemctl is-active --quiet ochenstarik-smm-control.service + curl --fail --silent --show-error \ + --retry 10 --retry-all-errors --retry-delay 1 \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + https://127.0.0.1:7443/healthz + + - name: Clean up systemd smoke installation + if: always() + run: sudo deploy/ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control || true -- 2.45.2 From 8f6d84069e492bc0bc7a516a223d10d11efcf1bb Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:00:04 +0700 Subject: [PATCH 07/31] Expose systemd smoke failure diagnostics --- .github/workflows/linux-control-agent.yml | 27 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 17c9c29..614195d 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -71,8 +71,18 @@ jobs: - name: Verify repeated systemd installation run: | set -Eeuo pipefail + smoke_log=/tmp/smm-systemd-smoke.log + report_failure() { + local title="$1" message + sudo systemctl status --no-pager ochenstarik-smm-control.service >>"$smoke_log" 2>&1 || true + sudo journalctl --no-pager -u ochenstarik-smm-control.service -n 40 >>"$smoke_log" 2>&1 || true + message="$(tail -c 6000 "$smoke_log" | sed ':a;N;$!ba;s/%/%25/g;s/\r/%0D/g;s/\n/%0A/g')" + printf '::error title=%s::%s\n' "$title" "$message" + exit 1 + } sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ - smoke-release.tar.gz 127.0.0.1 7443 + smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ + || report_failure 'Initial Control installation failed' for _ in {1..30}; do curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz && break @@ -80,15 +90,20 @@ jobs: done curl --fail --silent --show-error \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ - https://127.0.0.1:7443/healthz + https://127.0.0.1:7443/healthz \ + || report_failure 'Initial Control healthcheck failed' sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ - smoke-release.tar.gz 127.0.0.1 7443 - sudo systemctl restart ochenstarik-smm-control.service - sudo systemctl is-active --quiet ochenstarik-smm-control.service + smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ + || report_failure 'Repeated Control installation failed' + sudo systemctl restart ochenstarik-smm-control.service \ + || report_failure 'Control restart failed' + sudo systemctl is-active --quiet ochenstarik-smm-control.service \ + || report_failure 'Control is inactive after restart' curl --fail --silent --show-error \ --retry 10 --retry-all-errors --retry-delay 1 \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ - https://127.0.0.1:7443/healthz + https://127.0.0.1:7443/healthz \ + || report_failure 'Repeated Control healthcheck failed' - name: Clean up systemd smoke installation if: always() -- 2.45.2 From fd1f3d9df8d2bbb3961cf398772f5c3cdaea6fbc Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:03:27 +0700 Subject: [PATCH 08/31] Fix bootstrap os-release variable collision --- deploy/ochenstarik-server-monitor-manager.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 5f0f307..f1dcae2 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail IFS=$'\n\t' readonly PROGRAM="ochenstarik-server-monitor-manager" -readonly VERSION="0.2.0-dev" +readonly PROGRAM_VERSION="0.2.0-dev" readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager" readonly LIB_DIR="/usr/local/lib/ochenstarik-server-monitor-manager" readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager" @@ -850,7 +850,7 @@ main() { shift || true case "$action" in help|-h|--help) usage ;; - version|--version) printf '%s %s\n' "$PROGRAM" "$VERSION" ;; + version|--version) printf '%s %s\n' "$PROGRAM" "$PROGRAM_VERSION" ;; preflight) preflight ;; verify-release) [[ $# -eq 1 ]] || fail "verify-release requires ARCHIVE"; verify_release_payload "$1" ;; install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;; -- 2.45.2 From 94a83be0ca800ada87f4aeeb2c9f70d4c80ab4c5 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:07:42 +0700 Subject: [PATCH 09/31] Create policy helper directory during install --- deploy/ochenstarik-server-monitor-manager.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index f1dcae2..66a7334 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -481,6 +481,7 @@ EOF chown root:"$CONTROL_USER" "$ETC_DIR/control.env" chmod 0640 "$ETC_DIR/control.env" chmod 0644 "$ETC_DIR/control-public-url" + install -d -m 0755 "$(dirname "$POLICY_HELPER")" install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER" install -d -m 0755 "$LIB_DIR/bootstrap" install -m 0644 "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$LIB_DIR/bootstrap/$FIREWALL_UNIT" -- 2.45.2 From a68c90410ce4141dd9946291efc06b1e24e2be91 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:12:13 +0700 Subject: [PATCH 10/31] Capture systemd smoke healthcheck errors --- .github/workflows/linux-control-agent.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 614195d..705906a 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -90,7 +90,7 @@ jobs: done curl --fail --silent --show-error \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ - https://127.0.0.1:7443/healthz \ + https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \ || report_failure 'Initial Control healthcheck failed' sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ @@ -102,7 +102,7 @@ jobs: curl --fail --silent --show-error \ --retry 10 --retry-all-errors --retry-delay 1 \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ - https://127.0.0.1:7443/healthz \ + https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \ || report_failure 'Repeated Control healthcheck failed' - name: Clean up systemd smoke installation -- 2.45.2 From 25d1741a292f45795dce3d3d8db90078db936d59 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:16:13 +0700 Subject: [PATCH 11/31] Report Control listener diagnostics in CI --- .github/workflows/linux-control-agent.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 705906a..bdba09b 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -73,9 +73,16 @@ jobs: set -Eeuo pipefail smoke_log=/tmp/smm-systemd-smoke.log report_failure() { - local title="$1" message + local title="$1" message main_pid sudo systemctl status --no-pager ochenstarik-smm-control.service >>"$smoke_log" 2>&1 || true sudo journalctl --no-pager -u ochenstarik-smm-control.service -n 40 >>"$smoke_log" 2>&1 || true + sudo ss -ltnp >>"$smoke_log" 2>&1 || true + main_pid="$(sudo systemctl show -p MainPID --value ochenstarik-smm-control.service 2>/dev/null || true)" + if [[ "$main_pid" =~ ^[1-9][0-9]*$ ]]; then + sudo sh -c "tr '\0' '\n' >"$smoke_log" 2>&1 || true + sudo sh -c "printf 'wchan='; cat /proc/$main_pid/wchan" >>"$smoke_log" 2>&1 || true + fi message="$(tail -c 6000 "$smoke_log" | sed ':a;N;$!ba;s/%/%25/g;s/\r/%0D/g;s/\n/%0A/g')" printf '::error title=%s::%s\n' "$title" "$message" exit 1 -- 2.45.2 From 2de6778d9e8a7d689e64c25593cd2edd469150ef Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:19:54 +0700 Subject: [PATCH 12/31] Capture stalled Control process diagnostics --- .github/workflows/linux-control-agent.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index bdba09b..d854788 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -82,6 +82,12 @@ jobs: sudo sh -c "tr '\0' '\n' >"$smoke_log" 2>&1 || true sudo sh -c "printf 'wchan='; cat /proc/$main_pid/wchan" >>"$smoke_log" 2>&1 || true + sudo ls -l "/proc/$main_pid/fd" >>"$smoke_log" 2>&1 || true + sudo lsof -p "$main_pid" >>"$smoke_log" 2>&1 || true + if command -v strace >/dev/null 2>&1; then + sudo timeout 3 strace -f -p "$main_pid" -e trace=file,network,read,write,poll,select \ + >>"$smoke_log" 2>&1 || true + fi fi message="$(tail -c 6000 "$smoke_log" | sed ':a;N;$!ba;s/%/%25/g;s/\r/%0D/g;s/\n/%0A/g')" printf '::error title=%s::%s\n' "$title" "$message" -- 2.45.2 From 7c64601a745e7de430fe4bdf8435ed9315416b0f Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:23:10 +0700 Subject: [PATCH 13/31] Set service working directories --- .github/workflows/linux-control-agent.yml | 6 ------ deploy/ochenstarik-smm-agent.service | 1 + deploy/ochenstarik-smm-control.service | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index d854788..bdba09b 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -82,12 +82,6 @@ jobs: sudo sh -c "tr '\0' '\n' >"$smoke_log" 2>&1 || true sudo sh -c "printf 'wchan='; cat /proc/$main_pid/wchan" >>"$smoke_log" 2>&1 || true - sudo ls -l "/proc/$main_pid/fd" >>"$smoke_log" 2>&1 || true - sudo lsof -p "$main_pid" >>"$smoke_log" 2>&1 || true - if command -v strace >/dev/null 2>&1; then - sudo timeout 3 strace -f -p "$main_pid" -e trace=file,network,read,write,poll,select \ - >>"$smoke_log" 2>&1 || true - fi fi message="$(tail -c 6000 "$smoke_log" | sed ':a;N;$!ba;s/%/%25/g;s/\r/%0D/g;s/\n/%0A/g')" printf '::error title=%s::%s\n' "$title" "$message" diff --git a/deploy/ochenstarik-smm-agent.service b/deploy/ochenstarik-smm-agent.service index 3e7659f..a8eec71 100644 --- a/deploy/ochenstarik-smm-agent.service +++ b/deploy/ochenstarik-smm-agent.service @@ -8,6 +8,7 @@ Type=simple User=ochenstarik-smm-agent Group=ochenstarik-smm-agent EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env +WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/agent ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/agent/ochenstarik-smm-agent Restart=on-failure RestartSec=10s diff --git a/deploy/ochenstarik-smm-control.service b/deploy/ochenstarik-smm-control.service index d4dac12..cd18b5a 100644 --- a/deploy/ochenstarik-smm-control.service +++ b/deploy/ochenstarik-smm-control.service @@ -8,6 +8,7 @@ Type=simple User=ochenstarik-smm-control Group=ochenstarik-smm-control EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env +WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/control ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control Restart=on-failure RestartSec=10s -- 2.45.2 From f6192a16e19ae07b819be65c950d990343d8b050 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:26:59 +0700 Subject: [PATCH 14/31] Run protected CA healthcheck as root --- .github/workflows/linux-control-agent.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index bdba09b..80617ef 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -91,11 +91,11 @@ jobs: smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ || report_failure 'Initial Control installation failed' for _ in {1..30}; do - curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + sudo curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz && break sleep 1 done - curl --fail --silent --show-error \ + sudo curl --fail --silent --show-error \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \ || report_failure 'Initial Control healthcheck failed' @@ -106,7 +106,7 @@ jobs: || report_failure 'Control restart failed' sudo systemctl is-active --quiet ochenstarik-smm-control.service \ || report_failure 'Control is inactive after restart' - curl --fail --silent --show-error \ + sudo curl --fail --silent --show-error \ --retry 10 --retry-all-errors --retry-delay 1 \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \ -- 2.45.2 From 470034f1376026eda5a52802b4080feb352357ec Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:38:05 +0700 Subject: [PATCH 15/31] Add local Mesh emergency recovery --- .github/workflows/linux-control-agent.yml | 7 + .github/workflows/linux-release.yml | 3 + deploy/ochenstarik-server-monitor-manager.sh | 8 ++ deploy/ochenstarik-smm-emergency | 129 +++++++++++++++++++ docs/installer-contract.md | 8 +- docs/linux-bootstrap.md | 13 ++ docs/roadmap.md | 2 +- tests/bootstrap/test-bootstrap-contract.sh | 9 ++ 8 files changed, 175 insertions(+), 4 deletions(-) create mode 100755 deploy/ochenstarik-smm-emergency diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 80617ef..709fa4b 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -41,8 +41,10 @@ jobs: run: | bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply + bash -n deploy/ochenstarik-smm-emergency shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-smm-emergency bash tests/bootstrap/test-bootstrap-contract.sh - name: Publish agent amd64 @@ -64,6 +66,7 @@ jobs: install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/ + install -m 0755 deploy/ochenstarik-smm-emergency smoke/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/ tar -C smoke -czf smoke-release.tar.gz agent control deploy bootstrap sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256 @@ -90,6 +93,10 @@ jobs: sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ || report_failure 'Initial Control installation failed' + sudo test -x /usr/local/sbin/ochenstarik-smm-emergency \ + || report_failure 'Emergency recovery command was not installed' + sudo /usr/local/sbin/ochenstarik-smm-emergency status >>"$smoke_log" 2>&1 \ + || report_failure 'Emergency recovery status failed' for _ in {1..30}; do sudo curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz && break diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index 5ab5706..c2f3e87 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -20,8 +20,10 @@ jobs: run: | bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply + bash -n deploy/ochenstarik-smm-emergency shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-smm-emergency bash tests/bootstrap/test-bootstrap-contract.sh - name: Package bootstrap @@ -87,6 +89,7 @@ jobs: install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ + install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz" tar -C out -czf "$archive" agent control deploy bootstrap diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 66a7334..336acc7 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -13,6 +13,7 @@ readonly AGENT_USER="ochenstarik-smm-agent" readonly CONTROL_UNIT="ochenstarik-smm-control.service" readonly AGENT_UNIT="ochenstarik-smm-agent.service" readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply" +readonly EMERGENCY_COMMAND="/usr/local/sbin/ochenstarik-smm-emergency" readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control" readonly MESH_DIR="${STATE_DIR}/mesh" readonly WG_DIR="${ETC_DIR}/wireguard" @@ -152,6 +153,8 @@ extract_archive() { [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing from archive." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency command is missing from archive." } verify_release_payload() { @@ -162,6 +165,7 @@ verify_release_payload() { [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency recovery command is missing." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing." [[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing." log "Release archive and checksum are valid." @@ -483,6 +487,8 @@ EOF chmod 0644 "$ETC_DIR/control-public-url" install -d -m 0755 "$(dirname "$POLICY_HELPER")" install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER" + install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" + install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND" install -d -m 0755 "$LIB_DIR/bootstrap" install -m 0644 "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$LIB_DIR/bootstrap/$FIREWALL_UNIT" printf '%s\n' "$CONTROL_USER ALL=(root) NOPASSWD: $POLICY_HELPER *" >"$SUDOERS_FILE" @@ -530,6 +536,8 @@ install_agent() { install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR" install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent" install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root" + install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" + install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND" if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then install -m 0600 -o "$AGENT_USER" -g "$AGENT_USER" "$ca_cert" "$ETC_DIR/control-ca.crt" else diff --git a/deploy/ochenstarik-smm-emergency b/deploy/ochenstarik-smm-emergency new file mode 100755 index 0000000..51cbad9 --- /dev/null +++ b/deploy/ochenstarik-smm-emergency @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +readonly PROGRAM="ochenstarik-smm-emergency" +readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager" +readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager" +readonly MARKER="$STATE_DIR/mesh/emergency-disabled" +readonly CONTROL_UNIT="ochenstarik-smm-control.service" +readonly AGENT_UNIT="ochenstarik-smm-agent.service" +readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service" +readonly WIREGUARD_UNIT="wg-quick@smm0.service" +readonly NFT_FAMILY="inet" +readonly NFT_TABLE="ochenstarik_smm" + +fail() { printf '%s: %s\n' "$PROGRAM" "$*" >&2; exit 78; } +log() { printf '%s: %s\n' "$PROGRAM" "$*"; } + +usage() { + cat <<'EOF' +Local Server Monitor Manager emergency recovery + +Usage: + ochenstarik-smm-emergency status + ochenstarik-smm-emergency mesh-disable + ochenstarik-smm-emergency mesh-enable + ochenstarik-smm-emergency firewall-restore + +Commands use only Server Monitor Manager-owned units, interface and nftables +table. They do not modify the host SSH service or unrelated firewall rules. +EOF +} + +require_root() { + [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "this action must run as root (use sudo)" +} + +unit_state() { + local unit="$1" + if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then + systemctl is-active "$unit" 2>/dev/null || true + else + printf '%s\n' "not-installed" + fi +} + +show_status() { + local unit + for unit in "$CONTROL_UNIT" "$AGENT_UNIT" "$FIREWALL_UNIT" "$WIREGUARD_UNIT"; do + printf '%s: %s\n' "$unit" "$(unit_state "$unit")" + done + if command -v ip >/dev/null 2>&1 && ip link show smm0 >/dev/null 2>&1; then + printf '%s\n' "mesh-interface: present" + else + printf '%s\n' "mesh-interface: absent" + fi + if command -v nft >/dev/null 2>&1 && nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then + printf '%s\n' "mesh-firewall: loaded" + else + printf '%s\n' "mesh-firewall: absent" + fi + [[ -f "$MARKER" ]] && printf '%s\n' "emergency-lock: active" || printf '%s\n' "emergency-lock: inactive" + printf '%s\n' "backups:" + find "$STATE_DIR/bootstrap-backups" -maxdepth 1 -type f \ + \( -name '*.tar.gz' -o -name '*.empty' \) -printf ' %f\n' 2>/dev/null | sort -r | head -n 10 || true +} + +delete_project_firewall() { + if command -v nft >/dev/null 2>&1; then + nft delete table "$NFT_FAMILY" "$NFT_TABLE" 2>/dev/null || true + fi +} + +mesh_disable() { + require_root + systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true + systemctl disable --now "$FIREWALL_UNIT" 2>/dev/null || true + delete_project_firewall + install -d -m 0700 "$(dirname "$MARKER")" + printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$MARKER" + chmod 0600 "$MARKER" + log "Mesh disabled locally; Control and SSH were not changed." +} + +restore_project_firewall() { + require_root + [[ -f "$ETC_DIR/mesh.nft" ]] || fail "managed firewall configuration is missing" + command -v nft >/dev/null 2>&1 || fail "nft is unavailable" + if ! nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then + nft --check -f "$ETC_DIR/mesh.nft" + fi + delete_project_firewall + if ! nft -f "$ETC_DIR/mesh.nft"; then + systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true + fail "managed firewall restore failed; Mesh was disabled to fail closed" + fi + if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then + systemctl enable "$FIREWALL_UNIT" >/dev/null + fi + log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links." +} + +mesh_enable() { + require_root + [[ -f /etc/wireguard/smm0.conf ]] || fail "WireGuard smm0 configuration is missing" + if [[ -f "$ETC_DIR/mesh.nft" ]]; then + restore_project_firewall + fi + systemctl enable "$WIREGUARD_UNIT" >/dev/null + systemctl restart "$WIREGUARD_UNIT" + rm -f -- "$MARKER" + log "Mesh enabled locally." +} + +main() { + local action="${1:-help}" + shift || true + [[ $# -eq 0 ]] || fail "unexpected arguments" + case "$action" in + help|-h|--help) usage ;; + status) show_status ;; + mesh-disable) mesh_disable ;; + mesh-enable) mesh_enable ;; + firewall-restore) restore_project_firewall ;; + *) fail "unknown action: $action" ;; + esac +} + +main "$@" diff --git a/docs/installer-contract.md b/docs/installer-contract.md index 29dfb3a..88981a5 100644 --- a/docs/installer-contract.md +++ b/docs/installer-contract.md @@ -82,11 +82,13 @@ control device-code DEVICE_ID control node-code NODE_ID control automation-token AUTOMATION_ID SOURCE_NODE_ID emergency status -emergency vpn-disable -emergency ssh-restore BACKUP_ID -emergency firewall-restore BACKUP_ID +emergency mesh-disable +emergency mesh-enable +emergency firewall-restore ``` +Текущая emergency-команда покрывает уже реализованные Mesh/WireGuard и project-owned nftables resources. Действия `vpn-disable` и `ssh-restore BACKUP_ID` добавляются вместе с соответствующими provisioning-модулями и их root-only backup format; до появления этих модулей команда намеренно их отклоняет. + CLI является non-interactive, кроме локального ввода enrollment code и явных подтверждений опасного удаления. Машиночитаемый режим возвращает versioned JSON и стабильные exit codes. ## 6. Идемпотентность и обновление diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md index 9e32cbe..13e73c3 100644 --- a/docs/linux-bootstrap.md +++ b/docs/linux-bootstrap.md @@ -96,3 +96,16 @@ sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy ``` Update создаёт root-only backup перед заменой binaries и автоматически восстанавливает предыдущую версию, если сервис не запускается. Перед alpha-тестом на реальных серверах обязательно сохраните отдельную консольную/SSH-сессию и не закрывайте основной административный доступ firewall-правилами проекта. + +## Локальное аварийное восстановление + +Установщик размещает независимую от Control Hub команду `/usr/local/sbin/ochenstarik-smm-emergency`. Она принимает только фиксированные действия и управляет исключительно интерфейсом `smm0`, systemd units и nftables-таблицей Server Monitor Manager: + +```bash +sudo ochenstarik-smm-emergency status +sudo ochenstarik-smm-emergency mesh-disable +sudo ochenstarik-smm-emergency firewall-restore +sudo ochenstarik-smm-emergency mesh-enable +``` + +`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0665d4a..d661f84 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -100,7 +100,7 @@ - [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; - [x] реализовать nftables policy helper вместо временного deny-by-default helper; - [ ] добавить VM CI matrix, повторную установку и reboot; -- [ ] добавить локальную emergency recovery command. +- [x] добавить локальную emergency recovery command для текущих Mesh/firewall-компонентов. ## Этап 8 — Provisioning control plane diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 7b5f5fe..c1aed61 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -5,6 +5,7 @@ IFS=$'\n\t' root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" helper="$root/deploy/ochenstarik-smm-policy-apply" +emergency="$root/deploy/ochenstarik-smm-emergency" help_output="$(bash "$bootstrap" --help)" version_output="$(bash "$bootstrap" --version)" @@ -20,6 +21,9 @@ grep -Fq "node-code NODE_ID" <<<"$help_output" grep -Fq "verify-release ARCHIVE" <<<"$help_output" grep -Fq "node-token NODE_ID" <<<"$help_output" grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" +emergency_help="$(bash "$emergency" --help)" +grep -Fq 'mesh-disable' <<<"$emergency_help" +grep -Fq 'firewall-restore' <<<"$emergency_help" if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2 @@ -30,6 +34,10 @@ if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2 exit 1 fi +if bash "$emergency" mesh-disable >/dev/null 2>&1; then + printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2 + exit 1 +fi policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)" printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state" @@ -49,6 +57,7 @@ mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/d install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent" install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" +install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency" install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/" -- 2.45.2 From 08ccda8aa258f63dfe21e96662f8279759583ab5 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:49:15 +0700 Subject: [PATCH 16/31] Add multi-architecture Linux system matrix --- .github/workflows/linux-control-agent.yml | 4 + .github/workflows/linux-platform-matrix.yml | 149 ++++++++++++++++++ .github/workflows/linux-release.yml | 4 + docs/linux-platform-matrix.md | 20 +++ docs/roadmap.md | 6 +- tests/bootstrap/run-native-systemd-smoke.sh | 39 +++++ .../bootstrap/run-systemd-container-smoke.sh | 67 ++++++++ tests/bootstrap/systemd-container.Dockerfile | 14 ++ 8 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/linux-platform-matrix.yml create mode 100644 docs/linux-platform-matrix.md create mode 100755 tests/bootstrap/run-native-systemd-smoke.sh create mode 100755 tests/bootstrap/run-systemd-container-smoke.sh create mode 100644 tests/bootstrap/systemd-container.Dockerfile diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 709fa4b..425d55e 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -42,9 +42,13 @@ jobs: bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply bash -n deploy/ochenstarik-smm-emergency + bash -n tests/bootstrap/run-native-systemd-smoke.sh + bash -n tests/bootstrap/run-systemd-container-smoke.sh shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply shellcheck --severity=error deploy/ochenstarik-smm-emergency + shellcheck --severity=error tests/bootstrap/run-native-systemd-smoke.sh + shellcheck --severity=error tests/bootstrap/run-systemd-container-smoke.sh bash tests/bootstrap/test-bootstrap-contract.sh - name: Publish agent amd64 diff --git a/.github/workflows/linux-platform-matrix.yml b/.github/workflows/linux-platform-matrix.yml new file mode 100644 index 0000000..502224a --- /dev/null +++ b/.github/workflows/linux-platform-matrix.yml @@ -0,0 +1,149 @@ +name: Linux platform matrix + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/linux-platform-matrix.yml' + - 'deploy/**' + - 'src/ServerMonitorManager.Agent/**' + - 'src/ServerMonitorManager.Control/**' + - 'src/ServerMonitorManager.Core/**' + - 'tests/bootstrap/**' + push: + branches: [main] + paths: + - '.github/workflows/linux-platform-matrix.yml' + - 'deploy/**' + - 'src/ServerMonitorManager.Agent/**' + - 'src/ServerMonitorManager.Control/**' + - 'src/ServerMonitorManager.Core/**' + - 'tests/bootstrap/**' + +permissions: + contents: read + +concurrency: + group: linux-platform-matrix-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-archive: + name: Release archive (${{ matrix.runtime }}) + runs-on: ubuntu-24.04 + strategy: + matrix: + runtime: [linux-x64, linux-arm64] + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up .NET 10 + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Publish compatible release + run: | + set -Eeuo pipefail + dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj \ + --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/agent + dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \ + --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control + install -d out/deploy out/bootstrap + install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ + install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ + install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/ + install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ + archive='server-monitor-manager-${{ matrix.runtime }}.tar.gz' + tar -C out -czf "$archive" agent control deploy bootstrap + sha256sum "$archive" >"$archive.sha256" + + - name: Upload release archive + uses: actions/upload-artifact@v6 + with: + name: server-monitor-manager-${{ matrix.runtime }}-matrix + path: | + server-monitor-manager-${{ matrix.runtime }}.tar.gz + server-monitor-manager-${{ matrix.runtime }}.tar.gz.sha256 + + ubuntu-vm: + name: ${{ matrix.name }} native VM + needs: release-archive + strategy: + fail-fast: false + matrix: + include: + - name: Ubuntu 22.04 x64 + runner: ubuntu-22.04 + runtime: linux-x64 + - name: Ubuntu 24.04 x64 + runner: ubuntu-24.04 + runtime: linux-x64 + - name: Ubuntu 22.04 arm64 + runner: ubuntu-22.04-arm + runtime: linux-arm64 + - name: Ubuntu 24.04 arm64 + runner: ubuntu-24.04-arm + runtime: linux-arm64 + runs-on: ${{ matrix.runner }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download release archive + uses: actions/download-artifact@v8 + with: + name: server-monitor-manager-${{ matrix.runtime }}-matrix + path: artifacts + + - name: Install twice and restart services + run: | + bash tests/bootstrap/run-native-systemd-smoke.sh \ + 'artifacts/server-monitor-manager-${{ matrix.runtime }}.tar.gz' \ + deploy/ochenstarik-server-monitor-manager.sh + + debian-systemd: + name: ${{ matrix.name }} systemd restart + needs: release-archive + strategy: + fail-fast: false + matrix: + include: + - name: Debian 12 x64 + runner: ubuntu-24.04 + runtime: linux-x64 + image: debian:12 + - name: Debian 13 x64 + runner: ubuntu-24.04 + runtime: linux-x64 + image: debian:13 + - name: Debian 12 arm64 + runner: ubuntu-24.04-arm + runtime: linux-arm64 + image: debian:12 + - name: Debian 13 arm64 + runner: ubuntu-24.04-arm + runtime: linux-arm64 + image: debian:13 + runs-on: ${{ matrix.runner }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download release archive + uses: actions/download-artifact@v8 + with: + name: server-monitor-manager-${{ matrix.runtime }}-matrix + path: artifacts + + - name: Install twice and restart systemd container + run: | + bash tests/bootstrap/run-systemd-container-smoke.sh \ + '${{ matrix.image }}' \ + 'artifacts/server-monitor-manager-${{ matrix.runtime }}.tar.gz' \ + deploy/ochenstarik-server-monitor-manager.sh diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index c2f3e87..8df7022 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -21,9 +21,13 @@ jobs: bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply bash -n deploy/ochenstarik-smm-emergency + bash -n tests/bootstrap/run-native-systemd-smoke.sh + bash -n tests/bootstrap/run-systemd-container-smoke.sh shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply shellcheck --severity=error deploy/ochenstarik-smm-emergency + shellcheck --severity=error tests/bootstrap/run-native-systemd-smoke.sh + shellcheck --severity=error tests/bootstrap/run-systemd-container-smoke.sh bash tests/bootstrap/test-bootstrap-contract.sh - name: Package bootstrap diff --git a/docs/linux-platform-matrix.md b/docs/linux-platform-matrix.md new file mode 100644 index 0000000..9f1db6d --- /dev/null +++ b/docs/linux-platform-matrix.md @@ -0,0 +1,20 @@ +# Linux platform matrix + +Workflow `linux-platform-matrix.yml` собирает те же self-contained release-архивы, которые устанавливает production bootstrap, отдельно для `linux-x64` и `linux-arm64`. + +## Реальные Ubuntu VM + +На GitHub-hosted VM проверяются: + +- Ubuntu 22.04 x64; +- Ubuntu 24.04 x64; +- Ubuntu 22.04 arm64; +- Ubuntu 24.04 arm64. + +Каждая VM выполняет `preflight`, проверяет checksum и содержимое архива, дважды устанавливает Control, проверяет emergency-команду, перезапускает systemd service и обращается к HTTPS `/healthz` через созданный CA. + +## Debian systemd containers + +Debian 12/13 проверяются для x64 и arm64 в privileged systemd-контейнерах на соответствующей архитектуре runner. Тест выполняет чистую и повторную установку, затем полностью перезапускает контейнер и проверяет автоматический запуск Control и HTTPS healthcheck после нового systemd boot. + +Systemd-контейнер проверяет дистрибутив, users, permissions, units, certificates, SQLite state и service lifecycle, но не считается полноценной Debian VM. Отдельным незакрытым критерием остаётся reboot настоящих Debian VM с собственным kernel. Физическая проверка WireGuard/nftables также выполняется по `three-server-acceptance.md`, поскольку GitHub runners не моделируют внешний NAT и cloud firewall. diff --git a/docs/roadmap.md b/docs/roadmap.md index d661f84..b918e17 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,6 +2,8 @@ Подробные требования к собственному bootstrap, управляемой настройке Linux и Xray находятся в [ТЗ Provisioning и Xray VPN](provisioning-vpn-requirements.md). Все компоненты Server Monitor Manager разрабатываются, версионируются и выпускаются только в этом репозитории. +Текущее автоматическое покрытие поддерживаемых Linux-платформ описано в [Linux platform matrix](linux-platform-matrix.md). + ## Этап 0 — граница и базовая архитектура - [x] переименовать проект в Server Monitor Manager; @@ -99,7 +101,9 @@ - [x] локально создавать Agent key/CSR, выполнять mTLS enrollment и не сохранять token; - [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; - [x] реализовать nftables policy helper вместо временного deny-by-default helper; -- [ ] добавить VM CI matrix, повторную установку и reboot; +- [x] добавить native Ubuntu 22.04/24.04 x64/arm64 VM CI и повторную установку; +- [x] добавить Debian 12/13 x64/arm64 systemd-container restart matrix; +- [ ] добавить полный reboot настоящих Debian VM; - [x] добавить локальную emergency recovery command для текущих Mesh/firewall-компонентов. ## Этап 8 — Provisioning control plane diff --git a/tests/bootstrap/run-native-systemd-smoke.sh b/tests/bootstrap/run-native-systemd-smoke.sh new file mode 100755 index 0000000..8789577 --- /dev/null +++ b/tests/bootstrap/run-native-systemd-smoke.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +archive="${1:?usage: run-native-systemd-smoke.sh ARCHIVE BOOTSTRAP}" +bootstrap="${2:?usage: run-native-systemd-smoke.sh ARCHIVE BOOTSTRAP}" +port="${SMM_SMOKE_PORT:-17443}" + +cleanup() { + sudo "$bootstrap" uninstall-control --confirm-destroy-control >/dev/null 2>&1 || true +} +trap cleanup EXIT + +sudo "$bootstrap" preflight +sudo "$bootstrap" verify-release "$archive" +sudo "$bootstrap" install-control "$archive" 127.0.0.1 "$port" +sudo test -x /usr/local/sbin/ochenstarik-smm-emergency +sudo /usr/local/sbin/ochenstarik-smm-emergency status + +for _ in {1..30}; do + if sudo curl --fail --silent \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + "https://127.0.0.1:$port/healthz" >/dev/null; then + break + fi + sleep 1 +done +sudo curl --fail --silent --show-error \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + "https://127.0.0.1:$port/healthz" + +sudo "$bootstrap" install-control "$archive" 127.0.0.1 "$port" +sudo systemctl restart ochenstarik-smm-control.service +sudo systemctl is-active --quiet ochenstarik-smm-control.service +sudo curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + "https://127.0.0.1:$port/healthz" + +printf '%s\n' "NATIVE_SYSTEMD_SMOKE=PASS" diff --git a/tests/bootstrap/run-systemd-container-smoke.sh b/tests/bootstrap/run-systemd-container-smoke.sh new file mode 100755 index 0000000..89eee9c --- /dev/null +++ b/tests/bootstrap/run-systemd-container-smoke.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +base_image="${1:?usage: run-systemd-container-smoke.sh BASE_IMAGE ARCHIVE BOOTSTRAP}" +archive="$(realpath "${2:?archive is required}")" +bootstrap="$(realpath "${3:?bootstrap is required}")" +name="smm-systemd-${RANDOM}-${RANDOM}" +image="smm-systemd-smoke:${base_image//[:\/]/-}" +port="17443" + +cleanup() { + docker rm -f "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker build \ + --build-arg "BASE_IMAGE=$base_image" \ + -f "$root/tests/bootstrap/systemd-container.Dockerfile" \ + -t "$image" \ + "$root" +docker run --detach --privileged --cgroupns=private \ + --tmpfs /run --tmpfs /run/lock --name "$name" "$image" >/dev/null + +for _ in {1..30}; do + if docker exec "$name" systemctl is-system-running >/dev/null 2>&1; then + break + fi + state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)" + [[ "$state" == "degraded" ]] && break + sleep 1 +done +state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)" +[[ "$state" == "running" || "$state" == "degraded" ]] || { + docker exec "$name" systemctl --failed --no-pager || true + printf '%s\n' "container systemd did not finish booting: $state" >&2 + exit 1 +} + +docker cp "$archive" "$name:/tmp/release.tar.gz" +docker cp "${archive}.sha256" "$name:/tmp/release.tar.gz.sha256" +docker cp "$bootstrap" "$name:/tmp/ochenstarik-server-monitor-manager.sh" +docker exec "$name" chmod 0700 /tmp/ochenstarik-server-monitor-manager.sh +docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh preflight +docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh install-control \ + /tmp/release.tar.gz 127.0.0.1 "$port" +docker exec "$name" curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + "https://127.0.0.1:$port/healthz" +docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh install-control \ + /tmp/release.tar.gz 127.0.0.1 "$port" + +docker restart "$name" >/dev/null +for _ in {1..60}; do + if docker exec "$name" systemctl is-active --quiet ochenstarik-smm-control.service; then + break + fi + sleep 1 +done +docker exec "$name" systemctl is-active --quiet ochenstarik-smm-control.service +docker exec "$name" curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \ + --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ + "https://127.0.0.1:$port/healthz" +docker exec "$name" /usr/local/sbin/ochenstarik-smm-emergency status + +printf '%s\n' "SYSTEMD_CONTAINER_SMOKE=PASS image=$base_image" diff --git a/tests/bootstrap/systemd-container.Dockerfile b/tests/bootstrap/systemd-container.Dockerfile new file mode 100644 index 0000000..3790429 --- /dev/null +++ b/tests/bootstrap/systemd-container.Dockerfile @@ -0,0 +1,14 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +ENV container=docker +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl iproute2 openssl procps sudo systemd systemd-sysv \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] -- 2.45.2 From 3a8d7da47f79d91d23fd46aea7adc154545f830b Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 15:08:29 +0700 Subject: [PATCH 17/31] Add provisioning job control plane --- README.md | 2 +- docs/roadmap.md | 11 +- .../ControlStore.cs | 58 +++- src/ServerMonitorManager.Control/Program.cs | 130 +++++++++ .../ProvisioningStore.cs | 271 ++++++++++++++++++ src/ServerMonitorManager.Core/Contracts.cs | 39 +++ .../SmmJsonContext.cs | 4 + .../ControlApiTests.cs | 42 +++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 73 +++++ 10 files changed, 623 insertions(+), 9 deletions(-) create mode 100644 src/ServerMonitorManager.Control/ProvisioningStore.cs diff --git a/README.md b/README.md index 0099910..6e85745 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. CI exercises the Control-to-helper process boundary, HTTP authorization, Agent parsing, Desktop contracts, and a 100-Node concurrent heartbeat/replay scenario. Still required are the project-owned bootstrap, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Provisioning/Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The first Provisioning control-plane slice now persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and exposes Operator-only create/read/confirm/cancel endpoints. CI exercises the Control-to-helper process boundary, HTTP authorization, Agent parsing, Desktop contracts, and a 100-Node concurrent heartbeat/replay scenario. Still required are Provisioning execution/retry/rollback and the restricted helper channel, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/docs/roadmap.md b/docs/roadmap.md index b918e17..c3275da 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -108,16 +108,19 @@ ## Этап 8 — Provisioning control plane -- [ ] модели и SQLite migrations для ProvisioningJob; +- [x] модели и SQLite migration v2 для ProvisioningJob; - [ ] state machine, confirmations, cancellation, retry и rollback; -- [ ] обязательные idempotency key, audit reason и job TTL; +- [x] создание, чтение, подтверждение и отмена через Operator API; +- [ ] выполнение, retry, verification и rollback в полной state machine; +- [x] обязательные idempotency key, audit reason и job TTL; - [ ] Agent job channel только для собственного `node_id`; -- [ ] versioned JSON schemas для каждого action type; +- [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; +- [ ] versioned JSON schemas для остальных action type; - [ ] restricted root helper через Unix socket; - [ ] structured redacted events и progress; - [ ] `NeedsReconciliation` после неопределённого результата; - [ ] desired/factual configuration и drift; -- [ ] запрет параллельных несовместимых опасных заданий. +- [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). ## Этап 9 — базовая настройка и пользователи diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index d56a832..caef29c 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -8,9 +8,9 @@ using ServerMonitorManager.Core; namespace ServerMonitorManager.Control; -public sealed class ControlStore(IOptions options) +public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 1; + private const int CurrentSchemaVersion = 2; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -124,9 +124,61 @@ public sealed class ControlStore(IOptions options) CREATE UNIQUE INDEX IF NOT EXISTS ux_links_active_policy ON links(source_node_id, target_node_id, protocol, port) WHERE desired_state = 'Active'; - PRAGMA user_version = 1; """; await command.ExecuteNonQueryAsync(cancellationToken); + + if (schemaVersion < 1) + { + var markVersionOne = connection.CreateCommand(); + markVersionOne.CommandText = "PRAGMA user_version = 1;"; + await markVersionOne.ExecuteNonQueryAsync(cancellationToken); + } + + if (schemaVersion < 2) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var migrateProvisioning = connection.CreateCommand(); + migrateProvisioning.Transaction = migration; + migrateProvisioning.CommandText = """ + CREATE TABLE IF NOT EXISTS provisioning_jobs ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL REFERENCES agents(node_id) ON DELETE CASCADE, + action_type TEXT NOT NULL, + schema_version INTEGER NOT NULL, + parameters_json TEXT NOT NULL, + state TEXT NOT NULL, + confirmation_required INTEGER NOT NULL, + audit_reason TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + confirmed_at TEXT NULL, + cancelled_at TEXT NULL, + version INTEGER NOT NULL, + last_error TEXT NULL + ); + CREATE INDEX IF NOT EXISTS ix_provisioning_jobs_node_created + ON provisioning_jobs(node_id, created_at DESC); + CREATE UNIQUE INDEX IF NOT EXISTS ux_provisioning_jobs_active_node + ON provisioning_jobs(node_id) + WHERE state NOT IN ('Completed', 'Cancelled', 'Failed', 'RolledBack', 'RollbackFailed'); + CREATE TABLE IF NOT EXISTS provisioning_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES provisioning_jobs(id) ON DELETE CASCADE, + recorded_at TEXT NOT NULL, + event_type TEXT NOT NULL, + state TEXT NOT NULL, + message TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS ix_provisioning_events_job_sequence + ON provisioning_events(job_id, sequence); + PRAGMA user_version = 2; + """; + await migrateProvisioning.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 21a3bc8..fcd7fc8 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -381,6 +381,76 @@ agents.MapPost("/heartbeat", async ( var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray())); +control.MapPost("/agents/{nodeId}/provisioning/jobs", async ( + string nodeId, + ProvisioningJobCreateRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!NodeIdValidator.IsValid(nodeId) || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = + ["Invalid node id, action schema, parameters, TTL, audit reason, or idempotency key."] + }); + } + + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var job = await controlStore.CreateProvisioningJobAsync( + nodeId, request, actor, cancellationToken); + return Results.Created($"/api/v1/control/provisioning/jobs/{job.Id}", job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningNodeNotFoundException) + { + return Results.NotFound(); + } + catch (SqliteException exception) when (exception.SqliteErrorCode == 19) + { + return Results.Conflict(new ProblemDetails + { + Title = "The node already has an incompatible active provisioning job." + }); + } +}); +control.MapGet("/provisioning/jobs/{id}", async ( + string id, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = ["Invalid provisioning job id."] + }); + } + var job = await controlStore.GetProvisioningJobAsync(id, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); +}); +control.MapPost("/provisioning/jobs/{id}/confirm", async ( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => + await ChangeProvisioningJobAsync( + id, request, context, controlStore, confirm: true, cancellationToken)); +control.MapPost("/provisioning/jobs/{id}/cancel", async ( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => + await ChangeProvisioningJobAsync( + id, request, context, controlStore, confirm: false, cancellationToken)); control.MapPost("/automations/token", async ( AutomationTokenCreateRequest request, HttpContext context, @@ -577,6 +647,41 @@ automation.MapGet("/links", async ( await app.RunAsync(); return 0; +static async Task ChangeProvisioningJobAsync( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + bool confirm, + CancellationToken cancellationToken) +{ + if (!ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."] + }); + } + + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var job = confirm + ? await controlStore.ConfirmProvisioningJobAsync(id, request, actor, cancellationToken) + : await controlStore.CancelProvisioningJobAsync(id, request, actor, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +} + public partial class Program; internal static class NodeIdValidator @@ -611,3 +716,28 @@ internal static class CertificateReenrollmentValidator => request.Reason.Length is >= 1 and <= 200 && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); } + +internal static class ProvisioningJobValidator +{ + private const int MaximumParametersBytes = 16 * 1024; + + public static bool IsValid(ProvisioningJobCreateRequest request) + => request.SchemaVersion == 1 + && request.ActionType is "preflight" or "system.base-install" + && request.Parameters.ValueKind == JsonValueKind.Object + && !request.Parameters.EnumerateObject().Any() + && request.Parameters.GetRawText().Length <= MaximumParametersBytes + && request.TtlMinutes is >= 5 and <= 1440 + && request.AuditReason.Length is >= 1 and <= 256 + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + + public static bool IsValid(ProvisioningJobCommandRequest request) + => request.Reason.Length is >= 1 and <= 256 + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + + public static bool IsValidId(string id) + => id.Length == 32 && Guid.TryParseExact(id, "N", out _); + + public static bool RequiresConfirmation(string actionType) + => actionType == "system.base-install"; +} diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs new file mode 100644 index 0000000..35f0eed --- /dev/null +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -0,0 +1,271 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Control; + +public sealed partial class ControlStore +{ + public async Task CreateProvisioningJobAsync( + string nodeId, + ProvisioningJobCreateRequest request, + string actor, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-create:{actor}:{nodeId}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCreateRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var exists = connection.CreateCommand(); + exists.Transaction = transaction; + exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);"; + exists.Parameters.AddWithValue("$node", nodeId); + if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1) + { + throw new ProvisioningNodeNotFoundException(nodeId); + } + + var now = DateTimeOffset.UtcNow; + var confirmationRequired = ProvisioningJobValidator.RequiresConfirmation(request.ActionType); + var job = new ProvisioningJob( + Guid.NewGuid().ToString("N"), + nodeId, + request.ActionType, + request.SchemaVersion, + request.Parameters.Clone(), + confirmationRequired + ? ProvisioningJobStates.AwaitingConfirmation + : ProvisioningJobStates.Queued, + confirmationRequired, + request.AuditReason, + actor, + now, + now, + now.AddMinutes(request.TtlMinutes), + null, + null, + 1, + null); + + var insert = connection.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = """ + INSERT INTO provisioning_jobs( + id, node_id, action_type, schema_version, parameters_json, state, + confirmation_required, audit_reason, created_by, created_at, updated_at, + expires_at, confirmed_at, cancelled_at, version, last_error) + VALUES( + $id, $node, $action, $schema, $parameters, $state, + $confirmation, $reason, $actor, $created, $updated, + $expires, NULL, NULL, $version, NULL); + """; + AddProvisioningJobParameters(insert, job); + await insert.ExecuteNonQueryAsync(cancellationToken); + await WriteProvisioningEventAsync( + connection, transaction, job.Id, "job.created", job.State, + "Provisioning job accepted by the control plane.", now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, job, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, "provisioning.job.created", job.Id, + JsonSerializer.Serialize(new { job.NodeId, job.ActionType, job.SchemaVersion, job.AuditReason }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return job; + } + + public async Task GetProvisioningJobAsync( + string id, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var command = connection.CreateCommand(); + command.CommandText = "SELECT * FROM provisioning_jobs WHERE id = $id;"; + command.Parameters.AddWithValue("$id", id); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + return await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null; + } + + public Task ConfirmProvisioningJobAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + CancellationToken cancellationToken = default) + => TransitionProvisioningJobAsync( + id, request, actor, ProvisioningJobStates.AwaitingConfirmation, + ProvisioningJobStates.Queued, "job.confirmed", "provisioning.job.confirmed", + setConfirmedAt: true, cancellationToken); + + public Task CancelProvisioningJobAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + CancellationToken cancellationToken = default) + => TransitionProvisioningJobAsync( + id, request, actor, null, ProvisioningJobStates.Cancelled, + "job.cancelled", "provisioning.job.cancelled", + setConfirmedAt: false, cancellationToken); + + private async Task TransitionProvisioningJobAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + string? requiredState, + string targetState, + string eventType, + string auditAction, + bool setConfirmedAt, + CancellationToken cancellationToken) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"{eventType}:{actor}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + var allowed = requiredState is not null + ? current.State == requiredState + : current.State is ProvisioningJobStates.Queued or ProvisioningJobStates.AwaitingConfirmation; + if (!allowed || current.ExpiresAt <= DateTimeOffset.UtcNow) + { + throw new ProvisioningTransitionException(current.State, targetState); + } + + var now = DateTimeOffset.UtcNow; + var updated = current with + { + State = targetState, + UpdatedAt = now, + ConfirmedAt = setConfirmedAt ? now : current.ConfirmedAt, + CancelledAt = targetState == ProvisioningJobStates.Cancelled ? now : current.CancelledAt, + Version = current.Version + 1 + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, updated_at = $updated, confirmed_at = $confirmed, + cancelled_at = $cancelled, version = $version + WHERE id = $id AND version = $previous_version; + """; + AddProvisioningJobParameters(update, updated); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, targetState); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, eventType, targetState, request.Reason, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, auditAction, id, + JsonSerializer.Serialize(new { request.Reason, State = targetState }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + + private static async Task ReadProvisioningJobAsync( + SqliteConnection connection, + SqliteTransaction transaction, + string id, + CancellationToken cancellationToken) + { + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "SELECT * FROM provisioning_jobs WHERE id = $id;"; + command.Parameters.AddWithValue("$id", id); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + return await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null; + } + + private static ProvisioningJob ReadProvisioningJob(SqliteDataReader reader) + => new( + reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetInt32(3), + ParseParameters(reader.GetString(4)), reader.GetString(5), + reader.GetInt32(6) == 1, reader.GetString(7), reader.GetString(8), + DateTimeOffset.Parse(reader.GetString(9)), DateTimeOffset.Parse(reader.GetString(10)), + DateTimeOffset.Parse(reader.GetString(11)), + reader.IsDBNull(12) ? null : DateTimeOffset.Parse(reader.GetString(12)), + reader.IsDBNull(13) ? null : DateTimeOffset.Parse(reader.GetString(13)), + reader.GetInt64(14), reader.IsDBNull(15) ? null : reader.GetString(15)); + + private static JsonElement ParseParameters(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } + + private static void AddProvisioningJobParameters(SqliteCommand command, ProvisioningJob job) + { + command.Parameters.AddWithValue("$id", job.Id); + command.Parameters.AddWithValue("$node", job.NodeId); + command.Parameters.AddWithValue("$action", job.ActionType); + command.Parameters.AddWithValue("$schema", job.SchemaVersion); + command.Parameters.AddWithValue("$parameters", job.Parameters.GetRawText()); + command.Parameters.AddWithValue("$state", job.State); + command.Parameters.AddWithValue("$confirmation", job.ConfirmationRequired ? 1 : 0); + command.Parameters.AddWithValue("$reason", job.AuditReason); + command.Parameters.AddWithValue("$actor", job.CreatedBy); + command.Parameters.AddWithValue("$created", job.CreatedAt.ToString("O")); + command.Parameters.AddWithValue("$updated", job.UpdatedAt.ToString("O")); + command.Parameters.AddWithValue("$expires", job.ExpiresAt.ToString("O")); + command.Parameters.AddWithValue("$confirmed", job.ConfirmedAt is null ? DBNull.Value : job.ConfirmedAt.Value.ToString("O")); + command.Parameters.AddWithValue("$cancelled", job.CancelledAt is null ? DBNull.Value : job.CancelledAt.Value.ToString("O")); + command.Parameters.AddWithValue("$version", job.Version); + } + + private static async Task WriteProvisioningEventAsync( + SqliteConnection connection, + SqliteTransaction transaction, + string jobId, + string eventType, + string state, + string message, + DateTimeOffset recordedAt, + CancellationToken cancellationToken) + { + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + VALUES ($job, $recorded, $event, $state, $message); + """; + command.Parameters.AddWithValue("$job", jobId); + command.Parameters.AddWithValue("$recorded", recordedAt.ToString("O")); + command.Parameters.AddWithValue("$event", eventType); + command.Parameters.AddWithValue("$state", state); + command.Parameters.AddWithValue("$message", message); + await command.ExecuteNonQueryAsync(cancellationToken); + } +} + +public sealed class ProvisioningNodeNotFoundException(string nodeId) : Exception($"Node '{nodeId}' was not found."); + +public sealed class ProvisioningTransitionException(string state, string targetState) + : Exception($"Provisioning job cannot transition from '{state}' to '{targetState}'."); diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 42f1101..ad4d79b 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -1,3 +1,5 @@ +using System.Text.Json; + namespace ServerMonitorManager.Core; public sealed record EnrollmentRequest( @@ -140,3 +142,40 @@ public sealed record ControlEvent( string PayloadJson); public sealed record ControlError(string Error); + +public sealed record ProvisioningJobCreateRequest( + string ActionType, + int SchemaVersion, + JsonElement Parameters, + int TtlMinutes, + string AuditReason, + string IdempotencyKey); + +public sealed record ProvisioningJobCommandRequest( + string Reason, + string IdempotencyKey); + +public sealed record ProvisioningJob( + string Id, + string NodeId, + string ActionType, + int SchemaVersion, + JsonElement Parameters, + string State, + bool ConfirmationRequired, + string AuditReason, + string CreatedBy, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + DateTimeOffset ExpiresAt, + DateTimeOffset? ConfirmedAt, + DateTimeOffset? CancelledAt, + long Version, + string? LastError); + +public static class ProvisioningJobStates +{ + public const string Queued = "Queued"; + public const string AwaitingConfirmation = "AwaitingConfirmation"; + public const string Cancelled = "Cancelled"; +} diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 847bd3b..d58cb45 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -25,4 +25,8 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(LinkPolicy[]))] [JsonSerializable(typeof(ControlEvent))] [JsonSerializable(typeof(ControlError))] +[JsonSerializable(typeof(ProvisioningJobCreateRequest))] +[JsonSerializable(typeof(ProvisioningJobCommandRequest))] +[JsonSerializable(typeof(ProvisioningJob))] +[JsonSerializable(typeof(ProvisioningJob[]))] public sealed partial class SmmJsonContext : JsonSerializerContext; diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 0c6096b..0b11e8c 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -76,6 +76,48 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal("application/problem+json", response.Content.Headers.ContentType?.MediaType); } + [Fact] + public async Task OperatorCanCreateAndReadProvisioningJob() + { + var store = _factory.Services.GetRequiredService(); + var cancellationToken = TestContext.Current.CancellationToken; + var token = await store.CreateEnrollmentTokenAsync("home", TimeSpan.FromMinutes(10), cancellationToken); + await store.EnrollAsync( + new ServerMonitorManager.Core.EnrollmentRequest( + "home", token, "csr", Guid.NewGuid().ToString()), + () => new ServerMonitorManager.Control.IssuedCertificate( + "certificate", "ca", "F1E2", DateTimeOffset.UtcNow.AddDays(1)), + cancellationToken); + + using var anonymous = _factory.CreateClient(); + Assert.Equal(HttpStatusCode.Unauthorized, (await anonymous.GetAsync( + $"/api/v1/control/provisioning/jobs/{Guid.NewGuid():N}", cancellationToken)).StatusCode); + + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc"); + client.DefaultRequestHeaders.Add("X-Test-Role", "Operator"); + using var response = await client.PostAsJsonAsync( + "/api/v1/control/agents/home/provisioning/jobs", + new + { + actionType = "system.base-install", + schemaVersion = 1, + parameters = new { }, + ttlMinutes = 60, + auditReason = "API integration test", + idempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + var job = await response.Content.ReadFromJsonAsync( + cancellationToken); + Assert.NotNull(job); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.AwaitingConfirmation, job.State); + Assert.Equal(HttpStatusCode.OK, (await client.GetAsync( + $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode); + } + public async ValueTask DisposeAsync() => await _factory.DisposeAsync(); private sealed class ControlApiFactory : WebApplicationFactory diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 4c2757b..adcee40 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -105,7 +105,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(1L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(2L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index beb3c61..5036f1c 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Options; using Microsoft.Data.Sqlite; +using System.Text.Json; using ServerMonitorManager.Control; using ServerMonitorManager.Core; using Xunit; @@ -10,6 +11,78 @@ public sealed class ControlStoreTests : IAsyncDisposable { private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-tests-{Guid.NewGuid():N}"); + [Fact] + public async Task VersionOneDatabaseMigratesToProvisioningSchema() + { + Directory.CreateDirectory(_directory); + var databasePath = Path.Combine(_directory, "control.db"); + await using (var connection = new SqliteConnection($"Data Source={databasePath}")) + { + await connection.OpenAsync(TestContext.Current.CancellationToken); + var command = connection.CreateCommand(); + command.CommandText = "PRAGMA user_version = 1;"; + await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + var store = CreateStore(); + await store.InitializeAsync(TestContext.Current.CancellationToken); + + await using var migrated = new SqliteConnection($"Data Source={databasePath}"); + await migrated.OpenAsync(TestContext.Current.CancellationToken); + var version = migrated.CreateCommand(); + version.CommandText = "PRAGMA user_version;"; + Assert.Equal(2L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + var table = migrated.CreateCommand(); + table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; + Assert.Equal(16L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + } + + [Fact] + public async Task ProvisioningJobRequiresConfirmationIsIdempotentAndLocksNode() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = CreateStore(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "home", "A1B2", cancellationToken); + using var parameters = JsonDocument.Parse("{}"); + var request = new ProvisioningJobCreateRequest( + "system.base-install", 1, parameters.RootElement.Clone(), 60, + "Prepare test server", Guid.NewGuid().ToString()); + + var created = await store.CreateProvisioningJobAsync("home", request, "operator", cancellationToken); + var replay = await store.CreateProvisioningJobAsync("home", request, "operator", cancellationToken); + + Assert.Equal(created.Id, replay.Id); + Assert.Equal(created.State, replay.State); + Assert.Equal(ProvisioningJobStates.AwaitingConfirmation, created.State); + Assert.True(created.ConfirmationRequired); + await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync( + "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken)); + + var confirmation = new ProvisioningJobCommandRequest( + "Approved for test", Guid.NewGuid().ToString()); + var confirmed = await store.ConfirmProvisioningJobAsync( + created.Id, confirmation, "operator", cancellationToken); + var confirmationReplay = await store.ConfirmProvisioningJobAsync( + created.Id, confirmation, "operator", cancellationToken); + Assert.Equal(confirmed!.Id, confirmationReplay!.Id); + Assert.Equal(confirmed.State, confirmationReplay.State); + Assert.Equal(ProvisioningJobStates.Queued, confirmed.State); + Assert.NotNull(confirmed.ConfirmedAt); + + var cancelled = await store.CancelProvisioningJobAsync( + created.Id, + new ProvisioningJobCommandRequest("Test completed", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State); + Assert.NotNull(cancelled.CancelledAt); + + var next = await store.CreateProvisioningJobAsync( + "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken); + Assert.Equal(ProvisioningJobStates.AwaitingConfirmation, next.State); + } + [Fact] public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates() { -- 2.45.2 From 8620ade42e46efb83d3fd068ce9abdd36e5f9491 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 17:44:06 +0700 Subject: [PATCH 18/31] Stabilize Debian systemd smoke files --- .../bootstrap/run-systemd-container-smoke.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/bootstrap/run-systemd-container-smoke.sh b/tests/bootstrap/run-systemd-container-smoke.sh index 89eee9c..42d25e2 100755 --- a/tests/bootstrap/run-systemd-container-smoke.sh +++ b/tests/bootstrap/run-systemd-container-smoke.sh @@ -9,6 +9,9 @@ bootstrap="$(realpath "${3:?bootstrap is required}")" name="smm-systemd-${RANDOM}-${RANDOM}" image="smm-systemd-smoke:${base_image//[:\/]/-}" port="17443" +smoke_dir="/root/smm-smoke" +remote_archive="$smoke_dir/release.tar.gz" +remote_bootstrap="$smoke_dir/ochenstarik-server-monitor-manager.sh" cleanup() { docker rm -f "$name" >/dev/null 2>&1 || true @@ -38,18 +41,19 @@ state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)" exit 1 } -docker cp "$archive" "$name:/tmp/release.tar.gz" -docker cp "${archive}.sha256" "$name:/tmp/release.tar.gz.sha256" -docker cp "$bootstrap" "$name:/tmp/ochenstarik-server-monitor-manager.sh" -docker exec "$name" chmod 0700 /tmp/ochenstarik-server-monitor-manager.sh -docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh preflight -docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh install-control \ - /tmp/release.tar.gz 127.0.0.1 "$port" +docker exec "$name" install -d -m 0700 "$smoke_dir" +docker cp "$archive" "$name:$remote_archive" +docker cp "${archive}.sha256" "$name:${remote_archive}.sha256" +docker cp "$bootstrap" "$name:$remote_bootstrap" +docker exec "$name" chmod 0700 "$remote_bootstrap" +docker exec "$name" "$remote_bootstrap" preflight +docker exec "$name" "$remote_bootstrap" install-control \ + "$remote_archive" 127.0.0.1 "$port" docker exec "$name" curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ "https://127.0.0.1:$port/healthz" -docker exec "$name" /tmp/ochenstarik-server-monitor-manager.sh install-control \ - /tmp/release.tar.gz 127.0.0.1 "$port" +docker exec "$name" "$remote_bootstrap" install-control \ + "$remote_archive" 127.0.0.1 "$port" docker restart "$name" >/dev/null for _ in {1..60}; do -- 2.45.2 From 2757881a411ae96a9ab3e1ac5cc65ee9303b4012 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 19:37:22 +0700 Subject: [PATCH 19/31] Add node-scoped provisioning job channel --- docs/roadmap.md | 2 +- src/ServerMonitorManager.Control/Program.cs | 14 +++++ .../ProvisioningStore.cs | 54 ++++++++++++++++++- src/ServerMonitorManager.Core/Contracts.cs | 1 + .../ControlApiTests.cs | 44 +++++++++++++++ .../ControlStoreTests.cs | 29 ++++++++++ 6 files changed, 142 insertions(+), 2 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index c3275da..7750855 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -113,7 +113,7 @@ - [x] создание, чтение, подтверждение и отмена через Operator API; - [ ] выполнение, retry, verification и rollback в полной state machine; - [x] обязательные idempotency key, audit reason и job TTL; -- [ ] Agent job channel только для собственного `node_id`; +- [x] атомарный Agent job channel только для собственного `node_id`; - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; - [ ] restricted root helper через Unix socket; diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index fcd7fc8..e84c0fe 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -377,6 +377,20 @@ agents.MapPost("/heartbeat", async ( }); } }); +agents.MapGet("/provisioning/jobs/next", async ( + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) || !NodeIdValidator.IsValid(nodeId)) + { + return Results.Forbid(); + } + + var job = await controlStore.ClaimNextProvisioningJobAsync(nodeId, cancellationToken); + return job is null ? Results.NoContent() : Results.Ok(job); +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 35f0eed..0c20ad2 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -96,6 +96,56 @@ public sealed partial class ControlStore return await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null; } + public async Task ClaimNextProvisioningJobAsync( + string nodeId, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var now = DateTimeOffset.UtcNow; + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + UPDATE provisioning_jobs SET + state = $preflight, + updated_at = $now, + version = version + 1 + WHERE id = ( + SELECT id FROM provisioning_jobs + WHERE node_id = $node + AND state = $queued + AND expires_at > $now + ORDER BY created_at, id + LIMIT 1) + AND state = $queued + RETURNING *; + """; + command.Parameters.AddWithValue("$node", nodeId); + command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued); + command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight); + command.Parameters.AddWithValue("$now", now.ToString("O")); + ProvisioningJob? job; + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + job = await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null; + } + if (job is null) + { + await transaction.CommitAsync(cancellationToken); + return null; + } + + await WriteProvisioningEventAsync( + connection, transaction, job.Id, "job.claimed", job.State, + "Provisioning job claimed by its assigned Node.", now, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.job.claimed", job.Id, + JsonSerializer.Serialize(new { job.NodeId, job.ActionType, job.SchemaVersion }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return job; + } + public Task ConfirmProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -148,7 +198,9 @@ public sealed partial class ControlStore } var allowed = requiredState is not null ? current.State == requiredState - : current.State is ProvisioningJobStates.Queued or ProvisioningJobStates.AwaitingConfirmation; + : current.State is ProvisioningJobStates.Queued + or ProvisioningJobStates.Preflight + or ProvisioningJobStates.AwaitingConfirmation; if (!allowed || current.ExpiresAt <= DateTimeOffset.UtcNow) { throw new ProvisioningTransitionException(current.State, targetState); diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index ad4d79b..8622801 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -176,6 +176,7 @@ public sealed record ProvisioningJob( public static class ProvisioningJobStates { public const string Queued = "Queued"; + public const string Preflight = "Preflight"; public const string AwaitingConfirmation = "AwaitingConfirmation"; public const string Cancelled = "Cancelled"; } diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 0b11e8c..17a0749 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -118,6 +118,50 @@ public sealed class ControlApiTests : IAsyncDisposable $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode); } + [Fact] + public async Task AgentReceivesOnlyItsOwnProvisioningJob() + { + var nodeId = $"node-{Guid.NewGuid():N}"[..13]; + var store = _factory.Services.GetRequiredService(); + var cancellationToken = TestContext.Current.CancellationToken; + var token = await store.CreateEnrollmentTokenAsync(nodeId, TimeSpan.FromMinutes(10), cancellationToken); + await store.EnrollAsync( + new ServerMonitorManager.Core.EnrollmentRequest( + nodeId, token, "csr", Guid.NewGuid().ToString()), + () => new ServerMonitorManager.Control.IssuedCertificate( + "certificate", "ca", Guid.NewGuid().ToString("N"), DateTimeOffset.UtcNow.AddDays(1)), + cancellationToken); + using var parameters = System.Text.Json.JsonDocument.Parse("{}"); + var created = await store.CreateProvisioningJobAsync( + nodeId, + new ServerMonitorManager.Core.ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 60, + "API Agent isolation test", Guid.NewGuid().ToString()), + "windows-pc", + cancellationToken); + + using var other = _factory.CreateClient(); + other.DefaultRequestHeaders.Add("X-Test-Identity", "other-node"); + other.DefaultRequestHeaders.Add("X-Test-Role", "Agent"); + Assert.Equal(HttpStatusCode.NoContent, (await other.GetAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); + + using var assigned = _factory.CreateClient(); + assigned.DefaultRequestHeaders.Add("X-Test-Identity", nodeId); + assigned.DefaultRequestHeaders.Add("X-Test-Role", "Agent"); + var response = await assigned.GetAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var claimed = await response.Content.ReadFromJsonAsync( + cancellationToken); + Assert.NotNull(claimed); + Assert.Equal(created.Id, claimed.Id); + Assert.Equal(nodeId, claimed.NodeId); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State); + Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); + } + public async ValueTask DisposeAsync() => await _factory.DisposeAsync(); private sealed class ControlApiFactory : WebApplicationFactory diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index 5036f1c..87bba8c 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -83,6 +83,35 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.AwaitingConfirmation, next.State); } + [Fact] + public async Task ProvisioningJobCanBeClaimedOnlyOnceByAssignedNode() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = CreateStore(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "home", "C3D4", cancellationToken); + await EnrollAgentAsync(store, "other", "E5F6", cancellationToken); + using var parameters = JsonDocument.Parse("{}"); + var created = await store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 60, + "Inspect server", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + + Assert.Null(await store.ClaimNextProvisioningJobAsync("other", cancellationToken)); + var claims = await Task.WhenAll( + store.ClaimNextProvisioningJobAsync("home", cancellationToken), + store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + + var claimed = Assert.Single(claims, job => job is not null)!; + Assert.Equal(created.Id, claimed.Id); + Assert.Equal("home", claimed.NodeId); + Assert.Equal(ProvisioningJobStates.Preflight, claimed.State); + Assert.Null(Assert.Single(claims, job => job is null)); + } + [Fact] public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates() { -- 2.45.2 From 722f08455dbc79f51654e073b9be1efc3af2cf1b Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 21:35:30 +0700 Subject: [PATCH 20/31] Add provisioning progress state machine --- .../ControlStore.cs | 19 ++- src/ServerMonitorManager.Control/Program.cs | 54 ++++++++ .../ProvisioningStore.cs | 122 +++++++++++++++++- src/ServerMonitorManager.Core/Contracts.cs | 15 +++ .../SmmJsonContext.cs | 1 + .../ControlApiTests.cs | 23 ++++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 64 ++++++++- 8 files changed, 295 insertions(+), 5 deletions(-) diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index caef29c..bcb5370 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 2; + private const int CurrentSchemaVersion = 3; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -179,6 +179,23 @@ public sealed partial class ControlStore(IOptions options) await migrateProvisioning.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 3) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var migrateProgress = connection.CreateCommand(); + migrateProgress.Transaction = migration; + migrateProgress.CommandText = """ + ALTER TABLE provisioning_jobs + ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0; + ALTER TABLE provisioning_jobs + ADD COLUMN current_step TEXT NOT NULL DEFAULT ''; + PRAGMA user_version = 3; + """; + await migrateProgress.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index e84c0fe..31a6ed5 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -391,6 +391,41 @@ agents.MapGet("/provisioning/jobs/next", async ( var job = await controlStore.ClaimNextProvisioningJobAsync(nodeId, cancellationToken); return job is null ? Results.NoContent() : Results.Ok(job); }); +agents.MapPost("/provisioning/jobs/{id}/progress", async ( + string id, + ProvisioningJobProgressRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) + || !NodeIdValidator.IsValid(nodeId) + || !ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningProgress"] = + ["Invalid job id, state, progress, step, event code, message, or idempotency key."] + }); + } + + try + { + var job = await controlStore.ReportProvisioningProgressAsync( + nodeId, id, request, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => @@ -749,9 +784,28 @@ internal static class ProvisioningJobValidator => request.Reason.Length is >= 1 and <= 256 && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValid(ProvisioningJobProgressRequest request) + => request.State is ProvisioningJobStates.Preflight + or ProvisioningJobStates.Running + or ProvisioningJobStates.Verifying + or ProvisioningJobStates.Completed + or ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + && request.ProgressPercent is >= 0 and <= 100 + && IsSafeCode(request.Step, 64) + && IsSafeCode(request.EventCode, 64) + && request.Message.Length <= 512 + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValidId(string id) => id.Length == 32 && Guid.TryParseExact(id, "N", out _); public static bool RequiresConfirmation(string actionType) => actionType == "system.base-install"; + + private static bool IsSafeCode(string value, int maximumLength) + => value.Length is >= 1 && value.Length <= maximumLength + && value.All(character => character is >= 'a' and <= 'z' + or >= '0' and <= '9' + or '.' or '-' or '_'); } diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 0c20ad2..35c7484 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -54,6 +54,8 @@ public sealed partial class ControlStore null, null, 1, + 0, + "queued", null); var insert = connection.CreateCommand(); @@ -166,6 +168,96 @@ public sealed partial class ControlStore "job.cancelled", "provisioning.job.cancelled", setConfirmedAt: false, cancellationToken); + public async Task ReportProvisioningProgressAsync( + string nodeId, + string id, + ProvisioningJobProgressRequest request, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-progress:{nodeId}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobProgressRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null || !string.Equals(current.NodeId, nodeId, StringComparison.Ordinal)) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (!ProvisioningStateMachine.CanReport( + current.State, request.State, current.ProgressPercent, request.ProgressPercent)) + { + throw new ProvisioningTransitionException(current.State, request.State); + } + + var now = DateTimeOffset.UtcNow; + var updated = current with + { + State = request.State, + ProgressPercent = request.ProgressPercent, + CurrentStep = request.Step, + UpdatedAt = now, + Version = current.Version + 1, + LastError = request.State is ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + ? request.EventCode + : null + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, + progress_percent = $progress, + current_step = $step, + updated_at = $updated, + version = $version, + last_error = $error + WHERE id = $id AND node_id = $node AND version = $previous_version; + """; + update.Parameters.AddWithValue("$state", updated.State); + update.Parameters.AddWithValue("$progress", updated.ProgressPercent); + update.Parameters.AddWithValue("$step", updated.CurrentStep); + update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O")); + update.Parameters.AddWithValue("$version", updated.Version); + update.Parameters.AddWithValue("$error", (object?)updated.LastError ?? DBNull.Value); + update.Parameters.AddWithValue("$id", updated.Id); + update.Parameters.AddWithValue("$node", nodeId); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, request.State); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, request.EventCode, request.State, + request.Message, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.job.progress", id, + JsonSerializer.Serialize(new + { + request.State, + request.ProgressPercent, + request.Step, + request.EventCode + }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + private async Task TransitionProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -265,7 +357,8 @@ public sealed partial class ControlStore DateTimeOffset.Parse(reader.GetString(11)), reader.IsDBNull(12) ? null : DateTimeOffset.Parse(reader.GetString(12)), reader.IsDBNull(13) ? null : DateTimeOffset.Parse(reader.GetString(13)), - reader.GetInt64(14), reader.IsDBNull(15) ? null : reader.GetString(15)); + reader.GetInt64(14), reader.GetInt32(16), reader.GetString(17), + reader.IsDBNull(15) ? null : reader.GetString(15)); private static JsonElement ParseParameters(string json) { @@ -321,3 +414,30 @@ public sealed class ProvisioningNodeNotFoundException(string nodeId) : Exception public sealed class ProvisioningTransitionException(string state, string targetState) : Exception($"Provisioning job cannot transition from '{state}' to '{targetState}'."); + +internal static class ProvisioningStateMachine +{ + public static bool CanReport(string current, string target, int currentProgress, int targetProgress) + { + if (targetProgress < currentProgress || targetProgress is < 0 or > 100) + { + return false; + } + if (target is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation) + { + return current is ProvisioningJobStates.Preflight + or ProvisioningJobStates.Running + or ProvisioningJobStates.Verifying; + } + return (current, target) switch + { + (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true, + (ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true, + (ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true, + (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100, + _ => false + }; + } +} diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 8622801..dda378e 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -155,6 +155,14 @@ public sealed record ProvisioningJobCommandRequest( string Reason, string IdempotencyKey); +public sealed record ProvisioningJobProgressRequest( + string State, + int ProgressPercent, + string Step, + string EventCode, + string Message, + string IdempotencyKey); + public sealed record ProvisioningJob( string Id, string NodeId, @@ -171,6 +179,8 @@ public sealed record ProvisioningJob( DateTimeOffset? ConfirmedAt, DateTimeOffset? CancelledAt, long Version, + int ProgressPercent, + string CurrentStep, string? LastError); public static class ProvisioningJobStates @@ -178,5 +188,10 @@ public static class ProvisioningJobStates public const string Queued = "Queued"; public const string Preflight = "Preflight"; public const string AwaitingConfirmation = "AwaitingConfirmation"; + public const string Running = "Running"; + public const string Verifying = "Verifying"; + public const string Completed = "Completed"; + public const string Failed = "Failed"; + public const string NeedsReconciliation = "NeedsReconciliation"; public const string Cancelled = "Cancelled"; } diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index d58cb45..9e76310 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -27,6 +27,7 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ControlError))] [JsonSerializable(typeof(ProvisioningJobCreateRequest))] [JsonSerializable(typeof(ProvisioningJobCommandRequest))] +[JsonSerializable(typeof(ProvisioningJobProgressRequest))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] public sealed partial class SmmJsonContext : JsonSerializerContext; diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 17a0749..15e0139 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -160,6 +160,29 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State); Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync( "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); + + var progress = new + { + state = "Running", + progressPercent = 25, + step = "apply", + eventCode = "apply.started", + message = "Applying the approved plan.", + idempotencyKey = Guid.NewGuid().ToString() + }; + Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/progress", + progress, + cancellationToken)).StatusCode); + using var progressResponse = await assigned.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/progress", + progress, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, progressResponse.StatusCode); + var updated = await progressResponse.Content.ReadFromJsonAsync( + cancellationToken); + Assert.Equal(25, updated!.ProgressPercent); + Assert.Equal("apply", updated.CurrentStep); } public async ValueTask DisposeAsync() => await _factory.DisposeAsync(); diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index adcee40..f70a384 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -105,7 +105,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(2L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index 87bba8c..40be431 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,10 +31,10 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(2L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; - Assert.Equal(16L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); } [Fact] @@ -110,6 +110,66 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal("home", claimed.NodeId); Assert.Equal(ProvisioningJobStates.Preflight, claimed.State); Assert.Null(Assert.Single(claims, job => job is null)); + + var preflight = new ProvisioningJobProgressRequest( + ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress", + "Operating system detected.", Guid.NewGuid().ToString()); + Assert.Null(await store.ReportProvisioningProgressAsync( + "other", created.Id, preflight, cancellationToken)); + var firstProgress = await store.ReportProvisioningProgressAsync( + "home", created.Id, preflight, cancellationToken); + var replay = await store.ReportProvisioningProgressAsync( + "home", created.Id, preflight, cancellationToken); + Assert.Equal(firstProgress!.Id, replay!.Id); + Assert.Equal(10, replay.ProgressPercent); + Assert.Equal("inspect-os", replay.CurrentStep); + await Assert.ThrowsAsync(() => + store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Completed, + ProgressPercent = 100, + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken)); + + var running = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Running, + ProgressPercent = 40, + Step = "apply", + EventCode = "apply.started", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + var verifying = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Verifying, + ProgressPercent = 90, + Step = "verify", + EventCode = "verify.started", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + var completed = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Completed, + ProgressPercent = 100, + Step = "complete", + EventCode = "job.completed", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + Assert.Equal(ProvisioningJobStates.Running, running!.State); + Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State); + Assert.Equal(ProvisioningJobStates.Completed, completed!.State); } [Fact] -- 2.45.2 From 41d33a5ce012ea02e7d0c6d9493b6650895a626a Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 22:00:38 +0700 Subject: [PATCH 21/31] Reconcile expired provisioning jobs --- docs/roadmap.md | 2 +- .../ControlMaintenance.cs | 15 +++- .../ControlStore.cs | 39 ++++++++- src/ServerMonitorManager.Control/Program.cs | 31 +++++++ .../ProvisioningStore.cs | 80 +++++++++++++++++++ .../ControlMaintenanceTests.cs | 49 ++++++++++++ 6 files changed, 207 insertions(+), 9 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 7750855..bbd4e30 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -118,7 +118,7 @@ - [ ] versioned JSON schemas для остальных action type; - [ ] restricted root helper через Unix socket; - [ ] structured redacted events и progress; -- [ ] `NeedsReconciliation` после неопределённого результата; +- [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift; - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). diff --git a/src/ServerMonitorManager.Control/ControlMaintenance.cs b/src/ServerMonitorManager.Control/ControlMaintenance.cs index 10285c1..de3db45 100644 --- a/src/ServerMonitorManager.Control/ControlMaintenance.cs +++ b/src/ServerMonitorManager.Control/ControlMaintenance.cs @@ -10,7 +10,9 @@ public sealed record ControlMaintenanceResult( int MetricsDeleted, int IdempotencyDeleted, int AuditDeleted, - int TokensDeleted); + int TokensDeleted, + int ProvisioningJobsCancelled, + int ProvisioningJobsNeedingReconciliation); public sealed class LinkExpirationBackgroundService( LinkService links, @@ -65,15 +67,20 @@ public sealed class ControlMaintenanceBackgroundService( { var result = await store.MaintainAsync(timeProvider.GetUtcNow(), stoppingToken); await backups.CreateIfDueAsync(timeProvider.GetUtcNow(), stoppingToken); - if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted + result.TokensDeleted > 0) + if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted + + result.TokensDeleted + result.ProvisioningJobsCancelled + + result.ProvisioningJobsNeedingReconciliation > 0) { logger.LogInformation( "Control maintenance removed {Metrics} metrics, {Idempotency} replay records, " - + "{Audit} audit records, and {Tokens} enrollment tokens.", + + "{Audit} audit records, and {Tokens} enrollment tokens; cancelled {Cancelled} " + + "expired jobs and marked {Reconciliation} jobs for reconciliation.", result.MetricsDeleted, result.IdempotencyDeleted, result.AuditDeleted, - result.TokensDeleted); + result.TokensDeleted, + result.ProvisioningJobsCancelled, + result.ProvisioningJobsNeedingReconciliation); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index bcb5370..478f698 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -203,8 +203,37 @@ public sealed partial class ControlStore(IOptions options) CancellationToken cancellationToken = default) { await using var connection = await OpenAsync(cancellationToken); + await using var transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); var command = connection.CreateCommand(); + command.Transaction = transaction; command.CommandText = """ + INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + SELECT id, $now, 'job.expired', 'Cancelled', + 'Provisioning job expired before execution.' + FROM provisioning_jobs + WHERE expires_at <= $now + AND state IN ('Queued', 'AwaitingConfirmation'); + UPDATE provisioning_jobs SET + state = 'Cancelled', cancelled_at = $now, updated_at = $now, + current_step = 'expired', version = version + 1, + last_error = 'job.ttl_expired' + WHERE expires_at <= $now + AND state IN ('Queued', 'AwaitingConfirmation'); + SELECT changes(); + INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + SELECT id, $now, 'job.reconciliation.required', 'NeedsReconciliation', + 'Execution lease expired; factual state must be inspected.' + FROM provisioning_jobs + WHERE expires_at <= $now + AND state IN ('Preflight', 'Running', 'Verifying'); + UPDATE provisioning_jobs SET + state = 'NeedsReconciliation', updated_at = $now, + current_step = 'reconcile', version = version + 1, + last_error = 'job.ttl_expired' + WHERE expires_at <= $now + AND state IN ('Preflight', 'Running', 'Verifying'); + SELECT changes(); DELETE FROM metric_samples WHERE recorded_at < $metric_cutoff; SELECT changes(); DELETE FROM idempotency WHERE created_at < $idempotency_cutoff; @@ -225,24 +254,26 @@ public sealed partial class ControlStore(IOptions options) command.Parameters.AddWithValue( "$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O")); command.Parameters.AddWithValue("$now", now.ToString("O")); - var deleted = new int[6]; + var changes = new int[8]; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { - for (var index = 0; index < deleted.Length; index++) + for (var index = 0; index < changes.Length; index++) { if (await reader.ReadAsync(cancellationToken)) { - deleted[index] = reader.GetInt32(0); + changes[index] = reader.GetInt32(0); } await reader.NextResultAsync(cancellationToken); } } + await transaction.CommitAsync(cancellationToken); var optimize = connection.CreateCommand(); optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);"; await optimize.ExecuteNonQueryAsync(cancellationToken); return new ControlMaintenanceResult( - deleted[0], deleted[1], deleted[2], deleted[3] + deleted[4] + deleted[5]); + changes[2], changes[3], changes[4], changes[5] + changes[6] + changes[7], + changes[0], changes[1]); } public async Task BackupDatabaseAsync(string destinationPath, CancellationToken cancellationToken = default) diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 31a6ed5..ce82667 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -500,6 +500,37 @@ control.MapPost("/provisioning/jobs/{id}/cancel", async ( CancellationToken cancellationToken) => await ChangeProvisioningJobAsync( id, request, context, controlStore, confirm: false, cancellationToken)); +control.MapPost("/provisioning/jobs/{id}/retry", async ( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."] + }); + } + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var job = await controlStore.RetryProvisioningJobAsync( + id, request, actor, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); control.MapPost("/automations/token", async ( AutomationTokenCreateRequest request, HttpContext context, diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 35c7484..c5676bf 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -258,6 +258,86 @@ public sealed partial class ControlStore return updated; } + public async Task RetryProvisioningJobAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-retry:{actor}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (current.State is not (ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation)) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued); + } + + var now = DateTimeOffset.UtcNow; + var originalLifetime = current.ExpiresAt - current.CreatedAt; + var retryLifetime = originalLifetime < TimeSpan.FromMinutes(5) + ? TimeSpan.FromMinutes(5) + : originalLifetime; + var updated = current with + { + State = ProvisioningJobStates.Queued, + ProgressPercent = 0, + CurrentStep = "retry-queued", + UpdatedAt = now, + ExpiresAt = now.Add(retryLifetime), + Version = current.Version + 1, + LastError = null + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, progress_percent = 0, current_step = $step, + updated_at = $updated, expires_at = $expires, + version = $version, last_error = NULL + WHERE id = $id AND version = $previous_version; + """; + update.Parameters.AddWithValue("$state", updated.State); + update.Parameters.AddWithValue("$step", updated.CurrentStep); + update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O")); + update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O")); + update.Parameters.AddWithValue("$version", updated.Version); + update.Parameters.AddWithValue("$id", id); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, "job.retry.queued", updated.State, + request.Reason, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, "provisioning.job.retry", id, + JsonSerializer.Serialize(new { request.Reason }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + private async Task TransitionProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index f70a384..7d26549 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -1,6 +1,7 @@ using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using System.Text.Json; using ServerMonitorManager.Control; using ServerMonitorManager.Core; using Xunit; @@ -108,6 +109,54 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } + [Fact] + public async Task ExpiredProvisioningJobsAreCancelledOrRequireReconciliationAndCanRetry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var (store, _) = CreateServices(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "queued-node", "AABB", cancellationToken); + await EnrollAgentAsync(store, "running-node", "CCDD", cancellationToken); + using var parameters = JsonDocument.Parse("{}"); + var queued = await store.CreateProvisioningJobAsync( + "queued-node", + new ProvisioningJobCreateRequest( + "system.base-install", 1, parameters.RootElement.Clone(), 5, + "Await approval", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + var running = await store.CreateProvisioningJobAsync( + "running-node", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Inspect node", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken)); + + var result = await store.MaintainAsync( + running.ExpiresAt.AddSeconds(1), cancellationToken); + + Assert.Equal(1, result.ProvisioningJobsCancelled); + Assert.Equal(1, result.ProvisioningJobsNeedingReconciliation); + Assert.Equal(ProvisioningJobStates.Cancelled, + (await store.GetProvisioningJobAsync(queued.Id, cancellationToken))!.State); + var reconciliation = await store.GetProvisioningJobAsync(running.Id, cancellationToken); + Assert.Equal(ProvisioningJobStates.NeedsReconciliation, reconciliation!.State); + Assert.Equal("job.ttl_expired", reconciliation.LastError); + + var retried = await store.RetryProvisioningJobAsync( + running.Id, + new ProvisioningJobCommandRequest("Factual state checked", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.Equal(ProvisioningJobStates.Queued, retried!.State); + Assert.Equal(0, retried.ProgressPercent); + Assert.Null(retried.LastError); + Assert.True(retried.ExpiresAt > running.ExpiresAt); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken)); + } + [Fact] public async Task BackupCanRestoreDatabaseAndCertificateAuthority() { -- 2.45.2 From 3ae9f4717b9337a76950f10c73e9088c9ad6451f Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 22:10:42 +0700 Subject: [PATCH 22/31] Add provisioning rollback workflow --- docs/roadmap.md | 2 +- .../ControlStore.cs | 36 +++++- src/ServerMonitorManager.Control/Program.cs | 34 ++++++ .../ProvisioningStore.cs | 107 +++++++++++++++++- src/ServerMonitorManager.Core/Contracts.cs | 3 + .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 90 ++++++++++++++- 7 files changed, 262 insertions(+), 12 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index bbd4e30..192b6a7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -109,7 +109,7 @@ ## Этап 8 — Provisioning control plane - [x] модели и SQLite migration v2 для ProvisioningJob; -- [ ] state machine, confirmations, cancellation, retry и rollback; +- [x] state machine, confirmations, cancellation, retry и rollback; - [x] создание, чтение, подтверждение и отмена через Operator API; - [ ] выполнение, retry, verification и rollback в полной state machine; - [x] обязательные idempotency key, audit reason и job TTL; diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index 478f698..6bdfae8 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 3; + private const int CurrentSchemaVersion = 4; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -196,6 +196,23 @@ public sealed partial class ControlStore(IOptions options) await migrateProgress.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 4) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var strengthenProvisioningLock = connection.CreateCommand(); + strengthenProvisioningLock.Transaction = migration; + strengthenProvisioningLock.CommandText = """ + DROP INDEX IF EXISTS ux_provisioning_jobs_active_node; + CREATE UNIQUE INDEX ux_provisioning_jobs_active_node + ON provisioning_jobs(node_id) + WHERE state NOT IN ('Completed', 'Cancelled', 'RolledBack'); + PRAGMA user_version = 4; + """; + await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( @@ -234,6 +251,17 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); SELECT changes(); + INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation', + 'Rollback lease expired; factual state must be inspected.' + FROM provisioning_jobs + WHERE expires_at <= $now AND state = 'RollingBack'; + UPDATE provisioning_jobs SET + state = 'NeedsReconciliation', updated_at = $now, + current_step = 'rollback-reconcile', version = version + 1, + last_error = 'job.rollback.ttl_expired' + WHERE expires_at <= $now AND state = 'RollingBack'; + SELECT changes(); DELETE FROM metric_samples WHERE recorded_at < $metric_cutoff; SELECT changes(); DELETE FROM idempotency WHERE created_at < $idempotency_cutoff; @@ -254,7 +282,7 @@ public sealed partial class ControlStore(IOptions options) command.Parameters.AddWithValue( "$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O")); command.Parameters.AddWithValue("$now", now.ToString("O")); - var changes = new int[8]; + var changes = new int[9]; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { for (var index = 0; index < changes.Length; index++) @@ -272,8 +300,8 @@ public sealed partial class ControlStore(IOptions options) optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);"; await optimize.ExecuteNonQueryAsync(cancellationToken); return new ControlMaintenanceResult( - changes[2], changes[3], changes[4], changes[5] + changes[6] + changes[7], - changes[0], changes[1]); + changes[3], changes[4], changes[5], changes[6] + changes[7] + changes[8], + changes[0], changes[1] + changes[2]); } public async Task BackupDatabaseAsync(string destinationPath, CancellationToken cancellationToken = default) diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index ce82667..aae4368 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -531,6 +531,37 @@ control.MapPost("/provisioning/jobs/{id}/retry", async ( return Results.Conflict(new ProblemDetails { Title = exception.Message }); } }); +control.MapPost("/provisioning/jobs/{id}/rollback", async ( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."] + }); + } + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var job = await controlStore.StartProvisioningRollbackAsync( + id, request, actor, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); control.MapPost("/automations/token", async ( AutomationTokenCreateRequest request, HttpContext context, @@ -822,6 +853,9 @@ internal static class ProvisioningJobValidator or ProvisioningJobStates.Completed or ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollingBack + or ProvisioningJobStates.RolledBack + or ProvisioningJobStates.RollbackFailed && request.ProgressPercent is >= 0 and <= 100 && IsSafeCode(request.Step, 64) && IsSafeCode(request.EventCode, 64) diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index c5676bf..1269dc6 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -109,22 +109,29 @@ public sealed partial class ControlStore command.Transaction = transaction; command.CommandText = """ UPDATE provisioning_jobs SET - state = $preflight, + state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END, + current_step = CASE + WHEN state = $queued THEN 'preflight' + ELSE 'rollback' + END, updated_at = $now, version = version + 1 WHERE id = ( SELECT id FROM provisioning_jobs WHERE node_id = $node - AND state = $queued + AND (state = $queued + OR (state = $rolling_back AND current_step = 'rollback-queued')) AND expires_at > $now - ORDER BY created_at, id + ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id LIMIT 1) - AND state = $queued + AND (state = $queued + OR (state = $rolling_back AND current_step = 'rollback-queued')) RETURNING *; """; command.Parameters.AddWithValue("$node", nodeId); command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued); command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight); + command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack); command.Parameters.AddWithValue("$now", now.ToString("O")); ProvisioningJob? job; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) @@ -209,6 +216,7 @@ public sealed partial class ControlStore Version = current.Version + 1, LastError = request.State is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollbackFailed ? request.EventCode : null }; @@ -284,7 +292,8 @@ public sealed partial class ControlStore return null; } if (current.State is not (ProvisioningJobStates.Failed - or ProvisioningJobStates.NeedsReconciliation)) + or ProvisioningJobStates.NeedsReconciliation) + || current.CurrentStep == "rollback-reconcile") { throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued); } @@ -338,6 +347,88 @@ public sealed partial class ControlStore return updated; } + public async Task StartProvisioningRollbackAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-rollback:{actor}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (current.State is not (ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollbackFailed)) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack); + } + + var now = DateTimeOffset.UtcNow; + var lifetime = current.ExpiresAt - current.CreatedAt; + if (lifetime < TimeSpan.FromMinutes(5)) + { + lifetime = TimeSpan.FromMinutes(5); + } + var updated = current with + { + State = ProvisioningJobStates.RollingBack, + ProgressPercent = 0, + CurrentStep = "rollback-queued", + UpdatedAt = now, + ExpiresAt = now.Add(lifetime), + Version = current.Version + 1, + LastError = null + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, progress_percent = 0, current_step = $step, + updated_at = $updated, expires_at = $expires, + version = $version, last_error = NULL + WHERE id = $id AND version = $previous_version; + """; + update.Parameters.AddWithValue("$state", updated.State); + update.Parameters.AddWithValue("$step", updated.CurrentStep); + update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O")); + update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O")); + update.Parameters.AddWithValue("$version", updated.Version); + update.Parameters.AddWithValue("$id", id); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, "job.rollback.queued", updated.State, + request.Reason, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, "provisioning.job.rollback", id, + JsonSerializer.Serialize(new { request.Reason }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + private async Task TransitionProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -509,6 +600,10 @@ internal static class ProvisioningStateMachine or ProvisioningJobStates.Running or ProvisioningJobStates.Verifying; } + if (target == ProvisioningJobStates.RollbackFailed) + { + return current == ProvisioningJobStates.RollingBack; + } return (current, target) switch { (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true, @@ -517,6 +612,8 @@ internal static class ProvisioningStateMachine (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100, + (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true, + (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100, _ => false }; } diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index dda378e..a10a631 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -193,5 +193,8 @@ public static class ProvisioningJobStates public const string Completed = "Completed"; public const string Failed = "Failed"; public const string NeedsReconciliation = "NeedsReconciliation"; + public const string RollingBack = "RollingBack"; + public const string RolledBack = "RolledBack"; + public const string RollbackFailed = "RollbackFailed"; public const string Cancelled = "Cancelled"; } diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 7d26549..4954a16 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(4L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index 40be431..c21ab3a 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(4L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); @@ -172,6 +172,94 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.Completed, completed!.State); } + [Fact] + public async Task FailedJobRollbackIsNodeScopedRecoverableAndTerminal() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = CreateStore(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "home", "7788", cancellationToken); + using var parameters = JsonDocument.Parse("{}"); + var created = await store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Rollback test", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + var failed = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.Failed, 20, "inspect-os", "preflight.failed", + "Preflight failed.", Guid.NewGuid().ToString()), + cancellationToken); + Assert.Equal(ProvisioningJobStates.Failed, failed!.State); + await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Must remain blocked", Guid.NewGuid().ToString()), + "operator", + cancellationToken)); + + var rollbackRequest = new ProvisioningJobCommandRequest( + "Restore preflight state", Guid.NewGuid().ToString()); + var queued = await store.StartProvisioningRollbackAsync( + created.Id, rollbackRequest, "operator", cancellationToken); + var replay = await store.StartProvisioningRollbackAsync( + created.Id, rollbackRequest, "operator", cancellationToken); + Assert.Equal(queued!.Id, replay!.Id); + Assert.Equal("rollback-queued", queued.CurrentStep); + var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken); + Assert.Equal(ProvisioningJobStates.RollingBack, claimed!.State); + Assert.Equal("rollback", claimed.CurrentStep); + Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + + var maintenance = await store.MaintainAsync( + queued.ExpiresAt.AddSeconds(1), cancellationToken); + Assert.Equal(1, maintenance.ProvisioningJobsNeedingReconciliation); + var uncertain = await store.GetProvisioningJobAsync(created.Id, cancellationToken); + Assert.Equal(ProvisioningJobStates.NeedsReconciliation, uncertain!.State); + Assert.Equal("rollback-reconcile", uncertain.CurrentStep); + await Assert.ThrowsAsync(() => + store.RetryProvisioningJobAsync( + created.Id, + new ProvisioningJobCommandRequest("Unsafe retry", Guid.NewGuid().ToString()), + "operator", + cancellationToken)); + + await store.StartProvisioningRollbackAsync( + created.Id, + new ProvisioningJobCommandRequest("Resume rollback", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + var rolling = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RollingBack, 50, "restore", "rollback.progress", + "Restoring backup.", Guid.NewGuid().ToString()), + cancellationToken); + var rolledBack = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RolledBack, 100, "rollback-complete", "rollback.completed", + "Backup restored.", Guid.NewGuid().ToString()), + cancellationToken); + Assert.Equal(50, rolling!.ProgressPercent); + Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State); + Assert.Null(rolledBack.LastError); + var replacement = await store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Allowed after rollback", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.Equal(ProvisioningJobStates.Queued, replacement.State); + } + [Fact] public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates() { -- 2.45.2 From a15b0dd7b35df8b52216db34d04d5dac9bdaa133 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 22:34:05 +0700 Subject: [PATCH 23/31] Add redacted provisioning event history --- docs/roadmap.md | 2 +- .../ControlStore.cs | 36 +++++++++--- src/ServerMonitorManager.Control/Program.cs | 17 ++++++ .../ProvisioningStore.cs | 56 +++++++++++++++++-- src/ServerMonitorManager.Core/Contracts.cs | 10 ++++ .../SmmJsonContext.cs | 2 + .../ControlApiTests.cs | 7 +++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 11 +++- 9 files changed, 126 insertions(+), 17 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 192b6a7..11853bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -117,7 +117,7 @@ - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; - [ ] restricted root helper через Unix socket; -- [ ] structured redacted events и progress; +- [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift; - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index 6bdfae8..b9f1bef 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 4; + private const int CurrentSchemaVersion = 5; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -213,6 +213,23 @@ public sealed partial class ControlStore(IOptions options) await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 5) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var addStructuredEventFields = connection.CreateCommand(); + addStructuredEventFields.Transaction = migration; + addStructuredEventFields.CommandText = """ + ALTER TABLE provisioning_events + ADD COLUMN step TEXT NOT NULL DEFAULT ''; + ALTER TABLE provisioning_events + ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0; + PRAGMA user_version = 5; + """; + await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( @@ -225,9 +242,10 @@ public sealed partial class ControlStore(IOptions options) var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = """ - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.expired', 'Cancelled', - 'Provisioning job expired before execution.' + 'Provisioning job expired before execution.', 'expired', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state IN ('Queued', 'AwaitingConfirmation'); @@ -238,9 +256,11 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Queued', 'AwaitingConfirmation'); SELECT changes(); - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.reconciliation.required', 'NeedsReconciliation', - 'Execution lease expired; factual state must be inspected.' + 'Execution lease expired; factual state must be inspected.', + 'reconcile', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); @@ -251,9 +271,11 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); SELECT changes(); - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation', - 'Rollback lease expired; factual state must be inspected.' + 'Rollback lease expired; factual state must be inspected.', + 'rollback-reconcile', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state = 'RollingBack'; UPDATE provisioning_jobs SET diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index aae4368..44f0e45 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -484,6 +484,23 @@ control.MapGet("/provisioning/jobs/{id}", async ( var job = await controlStore.GetProvisioningJobAsync(id, cancellationToken); return job is null ? Results.NotFound() : Results.Ok(job); }); +control.MapGet("/provisioning/jobs/{id}/events", async ( + string id, + int? limit, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id) || limit is < 1 or > 200) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningEvents"] = ["Invalid provisioning job id or limit (1-200)."] + }); + } + var events = await controlStore.ListProvisioningEventsAsync( + id, limit ?? 100, cancellationToken); + return events is null ? Results.NotFound() : Results.Ok(events.ToArray()); +}); control.MapPost("/provisioning/jobs/{id}/confirm", async ( string id, ProvisioningJobCommandRequest request, diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 1269dc6..f6a81a8 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -248,7 +248,8 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, request.EventCode, request.State, - request.Message, now, cancellationToken); + $"Agent reported '{request.EventCode}' for step '{request.Step}' at " + + $"{request.ProgressPercent}%.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -266,6 +267,44 @@ public sealed partial class ControlStore return updated; } + public async Task?> ListProvisioningEventsAsync( + string id, + int limit, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var exists = connection.CreateCommand(); + exists.CommandText = "SELECT EXISTS(SELECT 1 FROM provisioning_jobs WHERE id = $id);"; + exists.Parameters.AddWithValue("$id", id); + if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1) + { + return null; + } + + var result = new List(); + var command = connection.CreateCommand(); + command.CommandText = """ + SELECT sequence, job_id, recorded_at, event_type, state, + step, progress_percent, message + FROM provisioning_events + WHERE job_id = $id + ORDER BY sequence DESC + LIMIT $limit; + """; + command.Parameters.AddWithValue("$id", id); + command.Parameters.AddWithValue("$limit", Math.Clamp(limit, 1, 200)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + result.Add(new ProvisioningEvent( + reader.GetInt64(0), reader.GetString(1), DateTimeOffset.Parse(reader.GetString(2)), + reader.GetString(3), reader.GetString(4), reader.GetString(5), + reader.GetInt32(6), reader.GetString(7))); + } + result.Reverse(); + return result; + } + public async Task RetryProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -336,7 +375,7 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, "job.retry.queued", updated.State, - request.Reason, now, cancellationToken); + "Operator queued a retry after reconciliation.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -418,7 +457,7 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, "job.rollback.queued", updated.State, - request.Reason, now, cancellationToken); + "Operator queued a rollback.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -494,7 +533,8 @@ public sealed partial class ControlStore } await WriteProvisioningEventAsync( - connection, transaction, id, eventType, targetState, request.Reason, now, cancellationToken); + connection, transaction, id, eventType, targetState, + $"Operator requested '{eventType}'.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -569,8 +609,12 @@ public sealed partial class ControlStore var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = """ - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) - VALUES ($job, $recorded, $event, $state, $message); + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) + SELECT $job, $recorded, $event, $state, $message, + current_step, progress_percent + FROM provisioning_jobs + WHERE id = $job; """; command.Parameters.AddWithValue("$job", jobId); command.Parameters.AddWithValue("$recorded", recordedAt.ToString("O")); diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index a10a631..689a60b 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -183,6 +183,16 @@ public sealed record ProvisioningJob( string CurrentStep, string? LastError); +public sealed record ProvisioningEvent( + long Sequence, + string JobId, + DateTimeOffset RecordedAt, + string EventType, + string State, + string Step, + int ProgressPercent, + string Message); + public static class ProvisioningJobStates { public const string Queued = "Queued"; diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 9e76310..04dc7fe 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -30,4 +30,6 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningJobProgressRequest))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] +[JsonSerializable(typeof(ProvisioningEvent))] +[JsonSerializable(typeof(ProvisioningEvent[]))] public sealed partial class SmmJsonContext : JsonSerializerContext; diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 15e0139..9ec249d 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -116,6 +116,13 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.AwaitingConfirmation, job.State); Assert.Equal(HttpStatusCode.OK, (await client.GetAsync( $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode); + using var eventsResponse = await client.GetAsync( + $"/api/v1/control/provisioning/jobs/{job.Id}/events?limit=10", cancellationToken); + Assert.Equal(HttpStatusCode.OK, eventsResponse.StatusCode); + var events = await eventsResponse.Content.ReadFromJsonAsync( + cancellationToken); + Assert.NotNull(events); + Assert.NotEmpty(events); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 4954a16..418d8cc 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(4L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(5L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index c21ab3a..f179c32 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(4L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(5L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); @@ -113,7 +113,7 @@ public sealed class ControlStoreTests : IAsyncDisposable var preflight = new ProvisioningJobProgressRequest( ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress", - "Operating system detected.", Guid.NewGuid().ToString()); + "secret-token-should-never-be-persisted", Guid.NewGuid().ToString()); Assert.Null(await store.ReportProvisioningProgressAsync( "other", created.Id, preflight, cancellationToken)); var firstProgress = await store.ReportProvisioningProgressAsync( @@ -170,6 +170,13 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.Running, running!.State); Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State); Assert.Equal(ProvisioningJobStates.Completed, completed!.State); + var events = await store.ListProvisioningEventsAsync(created.Id, 100, cancellationToken); + Assert.NotNull(events); + var preflightEvent = Assert.Single(events!, item => item.EventType == "preflight.progress"); + Assert.Equal("inspect-os", preflightEvent.Step); + Assert.Equal(10, preflightEvent.ProgressPercent); + Assert.DoesNotContain( + events!, item => item.Message.Contains("secret-token", StringComparison.Ordinal)); } [Fact] -- 2.45.2 From 4d08b93d237aecf4a7e4aa696e4c17fbfcf337fb Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 22:59:41 +0700 Subject: [PATCH 24/31] Add restricted provisioning preflight helper --- .github/workflows/linux-control-agent.yml | 12 +- .github/workflows/linux-platform-matrix.yml | 6 +- .github/workflows/linux-release.yml | 6 +- README.md | 2 +- ServerMonitorManager.slnx | 1 + deploy/ochenstarik-server-monitor-manager.sh | 33 ++++- deploy/ochenstarik-smm-agent.service | 3 +- ...chenstarik-smm-provisioning-helper.service | 31 ++++ docs/roadmap.md | 2 +- src/ServerMonitorManager.Agent/AgentClient.cs | 88 +++++++++++ .../AgentOptions.cs | 1 + src/ServerMonitorManager.Agent/Program.cs | 3 +- .../ProvisioningHelperClient.cs | 53 +++++++ src/ServerMonitorManager.Core/Contracts.cs | 32 ++++ .../SmmJsonContext.cs | 3 + .../Program.cs | 24 +++ .../ProvisioningHelperServer.cs | 139 ++++++++++++++++++ ...rMonitorManager.Provisioning.Helper.csproj | 14 ++ .../ProvisioningHelperTests.cs | 56 +++++++ .../ServerMonitorManager.Control.Tests.csproj | 1 + tests/bootstrap/test-bootstrap-contract.sh | 6 +- 21 files changed, 504 insertions(+), 12 deletions(-) create mode 100644 deploy/ochenstarik-smm-provisioning-helper.service create mode 100644 src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs create mode 100644 src/ServerMonitorManager.Provisioning.Helper/Program.cs create mode 100644 src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs create mode 100644 src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj create mode 100644 tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 425d55e..e442118 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -57,6 +57,12 @@ jobs: - name: Publish agent arm64 run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true + - name: Publish provisioning helper amd64 + run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime linux-x64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true + + - name: Publish provisioning helper arm64 + run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true + - name: Build systemd smoke release run: | dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj \ @@ -65,14 +71,18 @@ jobs: dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \ --configuration Release --runtime linux-x64 --self-contained true \ -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/control + dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj \ + --configuration Release --runtime linux-x64 --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/provisioning-helper install -d smoke/deploy smoke/bootstrap install -m 0644 deploy/ochenstarik-smm-control.service smoke/deploy/ install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/ + install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service smoke/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/ install -m 0755 deploy/ochenstarik-smm-emergency smoke/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/ - tar -C smoke -czf smoke-release.tar.gz agent control deploy bootstrap + tar -C smoke -czf smoke-release.tar.gz agent control provisioning-helper deploy bootstrap sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256 - name: Verify repeated systemd installation diff --git a/.github/workflows/linux-platform-matrix.yml b/.github/workflows/linux-platform-matrix.yml index 502224a..b018276 100644 --- a/.github/workflows/linux-platform-matrix.yml +++ b/.github/workflows/linux-platform-matrix.yml @@ -52,15 +52,19 @@ jobs: dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \ --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \ -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control + dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj \ + --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \ + -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/provisioning-helper install -d out/deploy out/bootstrap install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive='server-monitor-manager-${{ matrix.runtime }}.tar.gz' - tar -C out -czf "$archive" agent control deploy bootstrap + tar -C out -czf "$archive" agent control provisioning-helper deploy bootstrap sha256sum "$archive" >"$archive.sha256" - name: Upload release archive diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index 8df7022..e8b263a 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -84,6 +84,9 @@ jobs: - name: Publish control run: dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control + - name: Publish provisioning helper + run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/provisioning-helper + - name: Package shell: bash run: | @@ -91,12 +94,13 @@ jobs: install -d out/deploy out/bootstrap install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ + install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz" - tar -C out -czf "$archive" agent control deploy bootstrap + tar -C out -czf "$archive" agent control provisioning-helper deploy bootstrap sha256sum "$archive" > "$archive.sha256" - name: Upload artifact diff --git a/README.md b/README.md index 6e85745..75cf6e8 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The first Provisioning control-plane slice now persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and exposes Operator-only create/read/confirm/cancel endpoints. CI exercises the Control-to-helper process boundary, HTTP authorization, Agent parsing, Desktop contracts, and a 100-Node concurrent heartbeat/replay scenario. Still required are Provisioning execution/retry/rollback and the restricted helper channel, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper now accepts only versioned, module-hashed allowlisted requests through a local Unix socket; the first executable action is read-only Linux `preflight`. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/ServerMonitorManager.slnx b/ServerMonitorManager.slnx index 25c13cb..c25a2d1 100644 --- a/ServerMonitorManager.slnx +++ b/ServerMonitorManager.slnx @@ -2,5 +2,6 @@ + diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 336acc7..34825df 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -12,6 +12,7 @@ readonly CONTROL_USER="ochenstarik-smm-control" readonly AGENT_USER="ochenstarik-smm-agent" readonly CONTROL_UNIT="ochenstarik-smm-control.service" readonly AGENT_UNIT="ochenstarik-smm-agent.service" +readonly PROVISIONING_HELPER_UNIT="ochenstarik-smm-provisioning-helper.service" readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply" readonly EMERGENCY_COMMAND="/usr/local/sbin/ochenstarik-smm-emergency" readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control" @@ -138,7 +139,7 @@ verify_archive() { [[ -n "$entry" ]] || continue [[ "$entry" != /* && "$entry" != *".."* ]] || fail "Unsafe archive entry: $entry" case "$entry" in - agent|agent/*|control|control/*|deploy|deploy/*|bootstrap|bootstrap/*) ;; + agent|agent/*|control|control/*|provisioning-helper|provisioning-helper/*|deploy|deploy/*|bootstrap|bootstrap/*) ;; *) fail "Unexpected archive entry: $entry" ;; esac done < <(tar -tzf "$archive") @@ -152,6 +153,7 @@ extract_archive() { tar -xzf "$archive" -C "$TEMP_DIR" --no-same-owner --no-same-permissions [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." + [[ -f "$TEMP_DIR/deploy/$PROVISIONING_HELPER_UNIT" ]] || fail "Provisioning helper systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing from archive." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency command is missing from archive." @@ -164,6 +166,7 @@ verify_release_payload() { extract_archive "$archive" [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." + [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] || fail "Provisioning helper binary is missing." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency recovery command is missing." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing." @@ -229,9 +232,11 @@ create_backup() { agent) list=( "usr/local/lib/ochenstarik-server-monitor-manager/agent" + "usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper" "etc/ochenstarik-server-monitor-manager/agent.env" "etc/ochenstarik-server-monitor-manager/control-ca.crt" "etc/systemd/system/$AGENT_UNIT" + "etc/systemd/system/$PROVISIONING_HELPER_UNIT" ) ;; *) fail "Unknown backup role: $role" ;; @@ -531,11 +536,13 @@ install_agent() { openssl x509 -in "$ca_cert" -noout >/dev/null 2>&1 || fail "Invalid Control CA certificate." extract_archive "$archive" [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." + [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] || fail "Provisioning helper binary is missing." backup_id="$(create_backup agent)" ensure_system_user "$AGENT_USER" install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR" install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent" install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root" + install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root" install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND" if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then @@ -564,6 +571,8 @@ EOF chown root:"$AGENT_USER" "$ETC_DIR/control-ca.crt" chmod 0640 "$ETC_DIR/control-ca.crt" install_unit "$TEMP_DIR/deploy/$AGENT_UNIT" "$AGENT_UNIT" + install_unit "$TEMP_DIR/deploy/$PROVISIONING_HELPER_UNIT" "$PROVISIONING_HELPER_UNIT" + systemctl enable --now "$PROVISIONING_HELPER_UNIT" systemctl enable --now "$AGENT_UNIT" systemctl is-active --quiet "$AGENT_UNIT" || { systemctl status --no-pager "$AGENT_UNIT" >&2 || true @@ -661,8 +670,19 @@ update_role() { esac [[ -x "$TEMP_DIR/$role/$binary" ]] || fail "$role binary is missing." systemctl stop "$unit" + if [[ "$role" == "agent" ]]; then + systemctl stop "$PROVISIONING_HELPER_UNIT" 2>/dev/null || true + fi backup_id="$(create_backup "$role")" install_tree_atomic "$TEMP_DIR/$role" "$LIB_DIR/$role" "$user" + if [[ "$role" == "agent" ]]; then + [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] \ + || fail "Provisioning helper binary is missing." + install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root" + install_unit "$TEMP_DIR/deploy/$AGENT_UNIT" "$AGENT_UNIT" + install_unit "$TEMP_DIR/deploy/$PROVISIONING_HELPER_UNIT" "$PROVISIONING_HELPER_UNIT" + systemctl enable --now "$PROVISIONING_HELPER_UNIT" + fi systemctl start "$unit" if ! systemctl is-active --quiet "$unit"; then log "Update failed; restoring backup $backup_id" @@ -684,8 +704,14 @@ restore_backup() { case "$role" in control) unit="$CONTROL_UNIT" ;; agent) unit="$AGENT_UNIT" ;; *) fail "Unknown role: $role" ;; esac [[ -f "$archive" ]] || fail "Backup archive not found: $backup_id" systemctl stop "$unit" || true + if [[ "$role" == "agent" ]]; then + systemctl stop "$PROVISIONING_HELPER_UNIT" || true + fi tar -C / -xzf "$archive" systemctl daemon-reload + if [[ "$role" == "agent" ]]; then + systemctl start "$PROVISIONING_HELPER_UNIT" + fi systemctl start "$unit" systemctl is-active --quiet "$unit" || fail "Rollback restored files but service is not active." log "$role restored from $backup_id" @@ -825,8 +851,9 @@ uninstall_agent() { local purge="${1:-}" require_root systemctl disable --now "$AGENT_UNIT" 2>/dev/null || true - rm -f -- "/etc/systemd/system/$AGENT_UNIT" "$ETC_DIR/agent.env" - rm -rf -- "$LIB_DIR/agent" + systemctl disable --now "$PROVISIONING_HELPER_UNIT" 2>/dev/null || true + rm -f -- "/etc/systemd/system/$AGENT_UNIT" "/etc/systemd/system/$PROVISIONING_HELPER_UNIT" "$ETC_DIR/agent.env" + rm -rf -- "$LIB_DIR/agent" "$LIB_DIR/provisioning-helper" [[ "$purge" == "--purge" ]] && rm -rf -- "$STATE_DIR/agent" "$ETC_DIR/control-ca.crt" systemctl daemon-reload log "Agent removed${purge:+ ($purge)}." diff --git a/deploy/ochenstarik-smm-agent.service b/deploy/ochenstarik-smm-agent.service index a8eec71..f82bbbf 100644 --- a/deploy/ochenstarik-smm-agent.service +++ b/deploy/ochenstarik-smm-agent.service @@ -1,7 +1,8 @@ [Unit] Description=Ochenstarik Server Monitor Manager Agent -After=network-online.target +After=network-online.target ochenstarik-smm-provisioning-helper.service Wants=network-online.target +Requires=ochenstarik-smm-provisioning-helper.service [Service] Type=simple diff --git a/deploy/ochenstarik-smm-provisioning-helper.service b/deploy/ochenstarik-smm-provisioning-helper.service new file mode 100644 index 0000000..ecf3697 --- /dev/null +++ b/deploy/ochenstarik-smm-provisioning-helper.service @@ -0,0 +1,31 @@ +[Unit] +Description=Ochenstarik Server Monitor Manager restricted provisioning helper +After=local-fs.target + +[Service] +Type=simple +User=root +Group=ochenstarik-smm-agent +UMask=0007 +RuntimeDirectory=ochenstarik-server-monitor-manager +RuntimeDirectoryMode=0750 +WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper +ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper/ochenstarik-smm-provisioning-helper +Restart=on-failure +RestartSec=5s +NoNewPrivileges=true +PrivateTmp=true +PrivateDevices=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +LockPersonality=true +RestrictSUIDSGID=true +RestrictAddressFamilies=AF_UNIX +CapabilityBoundingSet= +AmbientCapabilities= + +[Install] +WantedBy=multi-user.target diff --git a/docs/roadmap.md b/docs/roadmap.md index 11853bc..2decf5d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -116,7 +116,7 @@ - [x] атомарный Agent job channel только для собственного `node_id`; - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; -- [ ] restricted root helper через Unix socket; +- [x] restricted root helper через Unix socket (первый allowlisted action `preflight`); - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift; diff --git a/src/ServerMonitorManager.Agent/AgentClient.cs b/src/ServerMonitorManager.Agent/AgentClient.cs index 013ddf6..032d73c 100644 --- a/src/ServerMonitorManager.Agent/AgentClient.cs +++ b/src/ServerMonitorManager.Agent/AgentClient.cs @@ -2,6 +2,7 @@ using System.Net; using System.Net.Http.Json; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; +using System.Text; using ServerMonitorManager.Core; namespace ServerMonitorManager.Agent; @@ -105,12 +106,99 @@ internal sealed class AgentClient(AgentOptions options) } } + try + { + await ExecuteNextProvisioningJobAsync(client, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + Console.Error.WriteLine($"Provisioning poll failed: {exception.Message}"); + } + var elapsed = DateTimeOffset.UtcNow - iterationStartedAt; var remaining = collectionDelay - elapsed; await Task.Delay(remaining > TimeSpan.Zero ? remaining : TimeSpan.FromSeconds(1), cancellationToken); } } + private async Task ExecuteNextProvisioningJobAsync( + HttpClient client, + CancellationToken cancellationToken) + { + using var response = await client.GetAsync("api/v1/agents/provisioning/jobs/next", cancellationToken); + if (response.StatusCode == HttpStatusCode.NoContent) + { + return; + } + response.EnsureSuccessStatusCode(); + var job = await response.Content.ReadFromJsonAsync( + SmmJsonContext.Default.ProvisioningJob, + cancellationToken) + ?? throw new InvalidOperationException("Control service returned an empty provisioning job."); + + if (job.ActionType != "preflight" || job.SchemaVersion != 1) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, + "dispatch", "action.unsupported", "Unsupported provisioning action.", cancellationToken); + return; + } + + var currentProgress = job.ProgressPercent; + try + { + var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath); + var result = await helper.RunPreflightAsync(job, cancellationToken); + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Running, 40, + "inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken); + currentProgress = 40; + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Verifying, 80, + "verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken); + currentProgress = 80; + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Completed, 100, + "completed", "preflight.completed", "Preflight completed.", cancellationToken); + Console.WriteLine( + $"Preflight {job.Id} completed: {result.OperatingSystem} " + + $"{result.OperatingSystemVersion} {result.Architecture}."); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, currentProgress, + "preflight", "preflight.failed", "Preflight helper failed.", cancellationToken); + Console.Error.WriteLine($"Preflight {job.Id} failed: {exception.Message}"); + } + } + + private static async Task ReportProvisioningAsync( + HttpClient client, + ProvisioningJob job, + string state, + int progress, + string step, + string eventCode, + string message, + CancellationToken cancellationToken) + { + var request = new ProvisioningJobProgressRequest( + state, progress, step, eventCode, message, CreateOperationId(job.Id, state)); + using var response = await client.PostAsJsonAsync( + $"api/v1/agents/provisioning/jobs/{job.Id}/progress", + request, + SmmJsonContext.Default.ProvisioningJobProgressRequest, + cancellationToken); + response.EnsureSuccessStatusCode(); + } + + private static string CreateOperationId(string jobId, string state) + { + var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}")); + return new Guid(digest.AsSpan(0, 16)).ToString(); + } + private static async Task SendHeartbeatAsync( HttpClient client, AgentHeartbeat heartbeat, diff --git a/src/ServerMonitorManager.Agent/AgentOptions.cs b/src/ServerMonitorManager.Agent/AgentOptions.cs index 68795da..b5603d5 100644 --- a/src/ServerMonitorManager.Agent/AgentOptions.cs +++ b/src/ServerMonitorManager.Agent/AgentOptions.cs @@ -6,6 +6,7 @@ public sealed class AgentOptions public Uri ControlUrl { get; init; } = new("https://127.0.0.1:7443"); public string StateDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/agent"; public string CertificateAuthorityPath { get; init; } = "/etc/ochenstarik-server-monitor-manager/control-ca.crt"; + public string ProvisioningSocketPath { get; init; } = "/run/ochenstarik-server-monitor-manager/provisioning.sock"; public int HeartbeatSeconds { get; init; } = 30; public int BufferMaxSamples { get; init; } = 720; public int BufferRecentSamples { get; init; } = 120; diff --git a/src/ServerMonitorManager.Agent/Program.cs b/src/ServerMonitorManager.Agent/Program.cs index 7e536fa..c9693a7 100644 --- a/src/ServerMonitorManager.Agent/Program.cs +++ b/src/ServerMonitorManager.Agent/Program.cs @@ -19,7 +19,8 @@ if (options.HeartbeatSeconds is < 10 or > 300 || options.BufferRecentSamples >= options.BufferMaxSamples || options.BufferDownsampleFactor is < 2 or > 100 || options.UploadBatchSize is < 1 or > 100 - || options.MaxRetrySeconds is < 10 or > 3600) + || options.MaxRetrySeconds is < 10 or > 3600 + || !Path.IsPathFullyQualified(options.ProvisioningSocketPath)) { Console.Error.WriteLine( "Invalid buffer settings: heartbeat 10-300s, max samples 10-10000, recent samples below max, " diff --git a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs new file mode 100644 index 0000000..48b13a8 --- /dev/null +++ b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs @@ -0,0 +1,53 @@ +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Agent; + +public sealed class ProvisioningHelperClient(string socketPath) +{ + private const int MaximumResponseBytes = 16 * 1024; + + public async Task RunPreflightAsync( + ProvisioningJob job, + CancellationToken cancellationToken) + { + var request = new ProvisioningHelperRequest( + "1", job.Id, job.ActionType, job.SchemaVersion, + ProvisioningActionCatalog.PreflightModuleHash, job.Parameters); + using var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); + await socket.ConnectAsync(new UnixDomainSocketEndPoint(socketPath), cancellationToken); + await using var stream = new NetworkStream(socket, ownsSocket: false); + var json = JsonSerializer.Serialize(request, SmmJsonContext.Default.ProvisioningHelperRequest) + "\n"; + await stream.WriteAsync(Encoding.UTF8.GetBytes(json), cancellationToken); + var payload = await ReadResponseAsync(stream, cancellationToken); + var response = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperResponse) + ?? throw new InvalidDataException("Provisioning helper returned an empty response."); + if (!response.Success || response.Preflight is null) + { + throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}"); + } + return response.Preflight; + } + + private static async Task ReadResponseAsync(Stream stream, CancellationToken cancellationToken) + { + using var buffer = new MemoryStream(); + var singleByte = new byte[1]; + while (buffer.Length <= MaximumResponseBytes) + { + var count = await stream.ReadAsync(singleByte, cancellationToken); + if (count == 0 || singleByte[0] == (byte)'\n') + { + break; + } + buffer.WriteByte(singleByte[0]); + } + if (buffer.Length == 0 || buffer.Length > MaximumResponseBytes) + { + throw new InvalidDataException("Provisioning helper response size is invalid."); + } + return buffer.ToArray(); + } +} diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 689a60b..307f6d2 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Serialization; namespace ServerMonitorManager.Core; @@ -163,6 +164,31 @@ public sealed record ProvisioningJobProgressRequest( string Message, string IdempotencyKey); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record ProvisioningHelperRequest( + string ProtocolVersion, + string JobId, + string ActionType, + int SchemaVersion, + string ModuleHash, + JsonElement Parameters); + +public sealed record ProvisioningHelperResponse( + bool Success, + string Code, + string Message, + ProvisioningPreflightResult? Preflight); + +public sealed record ProvisioningPreflightResult( + string OperatingSystem, + string OperatingSystemVersion, + string Architecture, + bool HasSystemd, + bool HasSshd, + bool HasNftables, + bool HasWireGuard, + bool HasApt); + public sealed record ProvisioningJob( string Id, string NodeId, @@ -208,3 +234,9 @@ public static class ProvisioningJobStates public const string RollbackFailed = "RollbackFailed"; public const string Cancelled = "Cancelled"; } + +public static class ProvisioningActionCatalog +{ + public const string PreflightModuleHash = + "2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b"; +} diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 04dc7fe..d65308d 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -28,6 +28,9 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningJobCreateRequest))] [JsonSerializable(typeof(ProvisioningJobCommandRequest))] [JsonSerializable(typeof(ProvisioningJobProgressRequest))] +[JsonSerializable(typeof(ProvisioningHelperRequest))] +[JsonSerializable(typeof(ProvisioningHelperResponse))] +[JsonSerializable(typeof(ProvisioningPreflightResult))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/src/ServerMonitorManager.Provisioning.Helper/Program.cs b/src/ServerMonitorManager.Provisioning.Helper/Program.cs new file mode 100644 index 0000000..0407ecc --- /dev/null +++ b/src/ServerMonitorManager.Provisioning.Helper/Program.cs @@ -0,0 +1,24 @@ +using ServerMonitorManager.Provisioning.Helper; + +if (!OperatingSystem.IsLinux()) +{ + Console.Error.WriteLine("The provisioning helper is supported only on Linux."); + return 2; +} + +if (args.Length != 0) +{ + Console.Error.WriteLine("The provisioning helper does not accept command-line arguments."); + return 2; +} +const string socketPath = "/run/ochenstarik-server-monitor-manager/provisioning.sock"; + +using var shutdown = new CancellationTokenSource(); +Console.CancelKeyPress += (_, eventArgs) => +{ + eventArgs.Cancel = true; + shutdown.Cancel(); +}; + +await new ProvisioningHelperServer(socketPath).RunAsync(shutdown.Token); +return 0; diff --git a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs new file mode 100644 index 0000000..55b89b5 --- /dev/null +++ b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs @@ -0,0 +1,139 @@ +using System.Net.Sockets; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using System.Text; +using System.Text.Json; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Provisioning.Helper; + +public sealed class ProvisioningHelperServer(string socketPath) +{ + private const int MaximumRequestBytes = 16 * 1024; + + [SupportedOSPlatform("linux")] + public async Task RunAsync(CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(socketPath)!); + File.Delete(socketPath); + using var listener = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); + listener.Bind(new UnixDomainSocketEndPoint(socketPath)); + File.SetUnixFileMode(socketPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite + | UnixFileMode.GroupRead | UnixFileMode.GroupWrite); + listener.Listen(8); + try + { + while (!cancellationToken.IsCancellationRequested) + { + var connection = await listener.AcceptAsync(cancellationToken); + await HandleAsync(connection, cancellationToken); + } + } + finally + { + File.Delete(socketPath); + } + } + + public static ProvisioningHelperResponse Execute(ProvisioningHelperRequest request) + { + if (request.ProtocolVersion != "1") + { + return Failure("protocol.unsupported", "Unsupported helper protocol version."); + } + if (request.JobId.Length != 32 || !request.JobId.All(Uri.IsHexDigit)) + { + return Failure("request.invalid-job", "Invalid provisioning job identifier."); + } + if (request.ActionType != "preflight" || request.SchemaVersion != 1 + || request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash + || request.Parameters.ValueKind != JsonValueKind.Object + || request.Parameters.EnumerateObject().Any()) + { + return Failure("action.denied", "The requested action is not allowed."); + } + + var release = ReadOperatingSystemRelease(); + var result = new ProvisioningPreflightResult( + release.GetValueOrDefault("ID", "linux"), + release.GetValueOrDefault("VERSION_ID", "unknown"), + RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(), + Directory.Exists("/run/systemd/system"), + Exists("/usr/sbin/sshd", "/usr/bin/sshd", "/sbin/sshd"), + Exists("/usr/sbin/nft", "/usr/bin/nft", "/sbin/nft"), + Exists("/usr/bin/wg", "/usr/sbin/wg", "/bin/wg"), + Exists("/usr/bin/apt-get", "/bin/apt-get")); + return new ProvisioningHelperResponse(true, "preflight.completed", "Preflight completed.", result); + } + + private static async Task HandleAsync(Socket socket, CancellationToken cancellationToken) + { + using (socket) + await using (var stream = new NetworkStream(socket, ownsSocket: false)) + { + ProvisioningHelperResponse response; + try + { + var payload = await ReadRequestAsync(stream, cancellationToken); + var request = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperRequest) + ?? throw new JsonException("Empty request."); + response = Execute(request); + } + catch (Exception exception) when (exception is JsonException or InvalidDataException) + { + response = Failure("request.invalid", "Invalid helper request."); + } + var json = JsonSerializer.Serialize(response, SmmJsonContext.Default.ProvisioningHelperResponse) + "\n"; + await stream.WriteAsync(Encoding.UTF8.GetBytes(json), cancellationToken); + } + } + + private static async Task ReadRequestAsync(Stream stream, CancellationToken cancellationToken) + { + using var buffer = new MemoryStream(); + var singleByte = new byte[1]; + while (buffer.Length <= MaximumRequestBytes) + { + var count = await stream.ReadAsync(singleByte, cancellationToken); + if (count == 0 || singleByte[0] == (byte)'\n') + { + break; + } + buffer.WriteByte(singleByte[0]); + } + if (buffer.Length == 0 || buffer.Length > MaximumRequestBytes) + { + throw new InvalidDataException("Request size is invalid."); + } + return buffer.ToArray(); + } + + private static Dictionary ReadOperatingSystemRelease() + { + var result = new Dictionary(StringComparer.Ordinal); + if (!File.Exists("/etc/os-release")) + { + return result; + } + foreach (var line in File.ReadLines("/etc/os-release")) + { + var separator = line.IndexOf('='); + if (separator <= 0) + { + continue; + } + var key = line[..separator]; + if (key is "ID" or "VERSION_ID") + { + result[key] = line[(separator + 1)..].Trim().Trim('"'); + } + } + return result; + } + + private static bool Exists(params string[] paths) => paths.Any(File.Exists); + + private static ProvisioningHelperResponse Failure(string code, string message) + => new(false, code, message, null); +} diff --git a/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj b/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj new file mode 100644 index 0000000..749100f --- /dev/null +++ b/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + true + 0.1.0 + ochenstarik-smm-provisioning-helper + + + + + diff --git a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs new file mode 100644 index 0000000..f5c8190 --- /dev/null +++ b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs @@ -0,0 +1,56 @@ +using System.Text.Json; +using ServerMonitorManager.Core; +using ServerMonitorManager.Provisioning.Helper; +using Xunit; + +namespace ServerMonitorManager.Control.Tests; + +public sealed class ProvisioningHelperTests +{ + [Fact] + public void HelperRejectsEveryActionOutsideFixedAllowlist() + { + using var document = JsonDocument.Parse("{}"); + var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest( + "1", new string('a', 32), "shell", 1, + ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone())); + + Assert.False(response.Success); + Assert.Equal("action.denied", response.Code); + Assert.Null(response.Preflight); + } + + [Fact] + public void HelperAcceptsOnlyEmptyPreflightSchemaOne() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + using var document = JsonDocument.Parse("{}"); + var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest( + "1", new string('b', 32), "preflight", 1, + ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone())); + + Assert.True(response.Success); + Assert.Equal("preflight.completed", response.Code); + Assert.NotNull(response.Preflight); + Assert.NotEmpty(response.Preflight.OperatingSystem); + Assert.NotEmpty(response.Preflight.Architecture); + } + + [Fact] + public void HelperContractRejectsUnknownJsonMembers() + { + const string json = """ + {"protocolVersion":"1","jobId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "actionType":"preflight","schemaVersion":1, + "moduleHash":"2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b", + "parameters":{},"command":"id"} + """; + + Assert.Throws(() => + JsonSerializer.Deserialize(json, SmmJsonContext.Default.ProvisioningHelperRequest)); + } +} diff --git a/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj b/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj index 9e9bb6b..5413588 100644 --- a/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj +++ b/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index c1aed61..135cdee 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -53,16 +53,18 @@ rm -f -- "$policy_state" fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)" trap 'rm -rf -- "$fixture"' EXIT -mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/deploy" "$fixture/payload/bootstrap" +mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap" install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent" install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" +install -m 0755 /bin/true "$fixture/payload/provisioning-helper/ochenstarik-smm-provisioning-helper" install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency" install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" +install -m 0644 "$root/deploy/ochenstarik-smm-provisioning-helper.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/" install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh" -tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control deploy bootstrap +tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control provisioning-helper deploy bootstrap sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256" bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null -- 2.45.2 From e0c755adf14665d13d264d5a86551c338d9f2f68 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 23:14:14 +0700 Subject: [PATCH 25/31] Persist typed provisioning preflight facts --- docs/roadmap.md | 2 +- src/ServerMonitorManager.Agent/AgentClient.cs | 51 ++++++--- .../ControlStore.cs | 23 +++- src/ServerMonitorManager.Control/Program.cs | 64 +++++++++++ .../ProvisioningFactsStore.cs | 100 ++++++++++++++++++ src/ServerMonitorManager.Core/Contracts.cs | 15 +++ .../SmmJsonContext.cs | 2 + .../ControlApiTests.cs | 48 +++++++++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 2 +- 10 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 src/ServerMonitorManager.Control/ProvisioningFactsStore.cs diff --git a/docs/roadmap.md b/docs/roadmap.md index 2decf5d..a994bb5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -119,7 +119,7 @@ - [x] restricted root helper через Unix socket (первый allowlisted action `preflight`); - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; -- [ ] desired/factual configuration и drift; +- [ ] desired/factual configuration и drift (типизированные factual preflight facts уже сохраняются); - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). ## Этап 9 — базовая настройка и пользователи diff --git a/src/ServerMonitorManager.Agent/AgentClient.cs b/src/ServerMonitorManager.Agent/AgentClient.cs index 032d73c..8d17822 100644 --- a/src/ServerMonitorManager.Agent/AgentClient.cs +++ b/src/ServerMonitorManager.Agent/AgentClient.cs @@ -144,33 +144,34 @@ internal sealed class AgentClient(AgentOptions options) return; } - var currentProgress = job.ProgressPercent; + ProvisioningPreflightResult result; try { var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath); - var result = await helper.RunPreflightAsync(job, cancellationToken); - await ReportProvisioningAsync( - client, job, ProvisioningJobStates.Running, 40, - "inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken); - currentProgress = 40; - await ReportProvisioningAsync( - client, job, ProvisioningJobStates.Verifying, 80, - "verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken); - currentProgress = 80; - await ReportProvisioningAsync( - client, job, ProvisioningJobStates.Completed, 100, - "completed", "preflight.completed", "Preflight completed.", cancellationToken); - Console.WriteLine( - $"Preflight {job.Id} completed: {result.OperatingSystem} " - + $"{result.OperatingSystemVersion} {result.Architecture}."); + result = await helper.RunPreflightAsync(job, cancellationToken); } catch (Exception exception) when (exception is not OperationCanceledException) { await ReportProvisioningAsync( - client, job, ProvisioningJobStates.Failed, currentProgress, + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, "preflight", "preflight.failed", "Preflight helper failed.", cancellationToken); Console.Error.WriteLine($"Preflight {job.Id} failed: {exception.Message}"); + return; } + + await ReportPreflightFactsAsync(client, job, result, cancellationToken); + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Running, 40, + "inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken); + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Verifying, 80, + "verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken); + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Completed, 100, + "completed", "preflight.completed", "Preflight completed.", cancellationToken); + Console.WriteLine( + $"Preflight {job.Id} completed: {result.OperatingSystem} " + + $"{result.OperatingSystemVersion} {result.Architecture}."); } private static async Task ReportProvisioningAsync( @@ -193,6 +194,22 @@ internal sealed class AgentClient(AgentOptions options) response.EnsureSuccessStatusCode(); } + private static async Task ReportPreflightFactsAsync( + HttpClient client, + ProvisioningJob job, + ProvisioningPreflightResult facts, + CancellationToken cancellationToken) + { + var request = new ProvisioningPreflightReportRequest( + facts, DateTimeOffset.UtcNow, CreateOperationId(job.Id, "facts")); + using var response = await client.PostAsJsonAsync( + $"api/v1/agents/provisioning/jobs/{job.Id}/preflight-facts", + request, + SmmJsonContext.Default.ProvisioningPreflightReportRequest, + cancellationToken); + response.EnsureSuccessStatusCode(); + } + private static string CreateOperationId(string jobId, string state) { var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}")); diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index b9f1bef..833ccbb 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 5; + private const int CurrentSchemaVersion = 6; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -230,6 +230,27 @@ public sealed partial class ControlStore(IOptions options) await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 6) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var addNodePreflightFacts = connection.CreateCommand(); + addNodePreflightFacts.Transaction = migration; + addNodePreflightFacts.CommandText = """ + CREATE TABLE IF NOT EXISTS node_preflight_facts ( + node_id TEXT PRIMARY KEY REFERENCES agents(node_id) ON DELETE CASCADE, + schema_version INTEGER NOT NULL, + facts_json TEXT NOT NULL, + observed_at TEXT NOT NULL, + source_job_id TEXT NOT NULL REFERENCES provisioning_jobs(id), + updated_at TEXT NOT NULL + ); + PRAGMA user_version = 6; + """; + await addNodePreflightFacts.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 44f0e45..0b7135a 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -426,10 +426,59 @@ agents.MapPost("/provisioning/jobs/{id}/progress", async ( return Results.Conflict(new ProblemDetails { Title = exception.Message }); } }); +agents.MapPost("/provisioning/jobs/{id}/preflight-facts", async ( + string id, + ProvisioningPreflightReportRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) + || !NodeIdValidator.IsValid(nodeId) + || !ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["preflightFacts"] = ["Invalid job id, facts, observation time, or idempotency key."] + }); + } + + try + { + var facts = await controlStore.RecordPreflightFactsAsync( + nodeId, id, request, cancellationToken); + return facts is null ? Results.NotFound() : Results.Ok(facts); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray())); +control.MapGet("/agents/{nodeId}/facts/preflight", async ( + string nodeId, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!NodeIdValidator.IsValid(nodeId)) + { + return Results.ValidationProblem(new Dictionary + { + ["nodeId"] = ["Invalid node id."] + }); + } + var facts = await controlStore.GetPreflightFactsAsync(nodeId, cancellationToken); + return facts is null ? Results.NotFound() : Results.Ok(facts); +}); control.MapPost("/agents/{nodeId}/provisioning/jobs", async ( string nodeId, ProvisioningJobCreateRequest request, @@ -879,6 +928,15 @@ internal static class ProvisioningJobValidator && request.Message.Length <= 512 && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValid(ProvisioningPreflightReportRequest request) + => request.ObservedAt >= DateTimeOffset.UtcNow.AddHours(-1) + && request.ObservedAt <= DateTimeOffset.UtcNow.AddMinutes(1) + && request.Facts is not null + && IsSafeFact(request.Facts.OperatingSystem, 32) + && IsSafeFact(request.Facts.OperatingSystemVersion, 64) + && request.Facts.Architecture is "x64" or "x86" or "arm" or "arm64" + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValidId(string id) => id.Length == 32 && Guid.TryParseExact(id, "N", out _); @@ -890,4 +948,10 @@ internal static class ProvisioningJobValidator && value.All(character => character is >= 'a' and <= 'z' or >= '0' and <= '9' or '.' or '-' or '_'); + + private static bool IsSafeFact(string? value, int maximumLength) + => value is not null + && value.Length is >= 1 && value.Length <= maximumLength + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '_' or '-' or '.'); } diff --git a/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs new file mode 100644 index 0000000..63ae71e --- /dev/null +++ b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs @@ -0,0 +1,100 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Control; + +public sealed partial class ControlStore +{ + public async Task RecordPreflightFactsAsync( + string nodeId, + string jobId, + ProvisioningPreflightReportRequest request, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-preflight:{nodeId}:{jobId}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningPreflightReportRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.NodePreflightFacts, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken); + if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal)) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (job.ActionType != "preflight" || job.SchemaVersion != 1 + || job.State != ProvisioningJobStates.Preflight) + { + throw new ProvisioningTransitionException(job.State, "PreflightFacts"); + } + + var facts = new NodePreflightFacts( + nodeId, 1, request.Facts, request.ObservedAt, jobId, DateTimeOffset.UtcNow); + var upsert = connection.CreateCommand(); + upsert.Transaction = transaction; + upsert.CommandText = """ + INSERT INTO node_preflight_facts( + node_id, schema_version, facts_json, observed_at, source_job_id, updated_at) + VALUES ($node, $schema, $facts, $observed, $job, $updated) + ON CONFLICT(node_id) DO UPDATE SET + schema_version = excluded.schema_version, + facts_json = excluded.facts_json, + observed_at = excluded.observed_at, + source_job_id = excluded.source_job_id, + updated_at = excluded.updated_at; + """; + upsert.Parameters.AddWithValue("$node", facts.NodeId); + upsert.Parameters.AddWithValue("$schema", facts.SchemaVersion); + upsert.Parameters.AddWithValue( + "$facts", JsonSerializer.Serialize(facts.Facts, SmmJsonContext.Default.ProvisioningPreflightResult)); + upsert.Parameters.AddWithValue("$observed", facts.ObservedAt.ToString("O")); + upsert.Parameters.AddWithValue("$job", facts.SourceJobId); + upsert.Parameters.AddWithValue("$updated", facts.UpdatedAt.ToString("O")); + await upsert.ExecuteNonQueryAsync(cancellationToken); + await WriteProvisioningEventAsync( + connection, transaction, jobId, "preflight.facts-recorded", job.State, + "Validated preflight facts were recorded.", facts.UpdatedAt, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, facts, + SmmJsonContext.Default.NodePreflightFacts, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.preflight.facts", jobId, + JsonSerializer.Serialize(new { facts.SchemaVersion, facts.ObservedAt }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return facts; + } + + public async Task GetPreflightFactsAsync( + string nodeId, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var command = connection.CreateCommand(); + command.CommandText = """ + SELECT schema_version, facts_json, observed_at, source_job_id, updated_at + FROM node_preflight_facts WHERE node_id = $node; + """; + command.Parameters.AddWithValue("$node", nodeId); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + var facts = JsonSerializer.Deserialize( + reader.GetString(1), SmmJsonContext.Default.ProvisioningPreflightResult) + ?? throw new InvalidDataException("Stored preflight facts are invalid."); + return new NodePreflightFacts( + nodeId, reader.GetInt32(0), facts, DateTimeOffset.Parse(reader.GetString(2)), + reader.GetString(3), DateTimeOffset.Parse(reader.GetString(4))); + } +} diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 307f6d2..1370bed 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -179,6 +179,7 @@ public sealed record ProvisioningHelperResponse( string Message, ProvisioningPreflightResult? Preflight); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] public sealed record ProvisioningPreflightResult( string OperatingSystem, string OperatingSystemVersion, @@ -189,6 +190,20 @@ public sealed record ProvisioningPreflightResult( bool HasWireGuard, bool HasApt); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record ProvisioningPreflightReportRequest( + ProvisioningPreflightResult Facts, + DateTimeOffset ObservedAt, + string IdempotencyKey); + +public sealed record NodePreflightFacts( + string NodeId, + int SchemaVersion, + ProvisioningPreflightResult Facts, + DateTimeOffset ObservedAt, + string SourceJobId, + DateTimeOffset UpdatedAt); + public sealed record ProvisioningJob( string Id, string NodeId, diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index d65308d..873e40e 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -31,6 +31,8 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningHelperRequest))] [JsonSerializable(typeof(ProvisioningHelperResponse))] [JsonSerializable(typeof(ProvisioningPreflightResult))] +[JsonSerializable(typeof(ProvisioningPreflightReportRequest))] +[JsonSerializable(typeof(NodePreflightFacts))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 9ec249d..be7aec1 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -168,6 +168,54 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync( "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); + var preflightReport = new + { + facts = new + { + operatingSystem = "ubuntu", + operatingSystemVersion = "24.04", + architecture = "x64", + hasSystemd = true, + hasSshd = true, + hasNftables = true, + hasWireGuard = false, + hasApt = true + }, + observedAt = DateTimeOffset.UtcNow, + idempotencyKey = Guid.NewGuid().ToString() + }; + Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts", + preflightReport, + cancellationToken)).StatusCode); + using var factsResponse = await assigned.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts", + preflightReport, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, factsResponse.StatusCode); + var recorded = await factsResponse.Content.ReadFromJsonAsync( + cancellationToken); + Assert.Equal(nodeId, recorded!.NodeId); + Assert.Equal("ubuntu", recorded.Facts.OperatingSystem); + using var replayResponse = await assigned.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts", + preflightReport, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, replayResponse.StatusCode); + var replayed = await replayResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal(recorded.UpdatedAt, replayed!.UpdatedAt); + + using var operatorClient = _factory.CreateClient(); + operatorClient.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc"); + operatorClient.DefaultRequestHeaders.Add("X-Test-Role", "Operator"); + using var storedFactsResponse = await operatorClient.GetAsync( + $"/api/v1/control/agents/{nodeId}/facts/preflight", cancellationToken); + Assert.Equal(HttpStatusCode.OK, storedFactsResponse.StatusCode); + var stored = await storedFactsResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal(created.Id, stored!.SourceJobId); + var progress = new { state = "Running", diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 418d8cc..528d5e0 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(5L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(6L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index f179c32..faf3efa 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(5L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(6L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); -- 2.45.2 From 3be4e77c6686d5476e5082767ae3fc794eb8c28e Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 23:32:07 +0700 Subject: [PATCH 26/31] Add preflight desired state drift detection --- README.md | 2 +- docs/roadmap.md | 2 +- .../ControlStore.cs | 24 ++- src/ServerMonitorManager.Control/Program.cs | 62 +++++++ .../ProvisioningFactsStore.cs | 172 ++++++++++++++++++ src/ServerMonitorManager.Core/Contracts.cs | 51 ++++++ .../SmmJsonContext.cs | 4 + .../ControlApiTests.cs | 71 ++++++++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 2 +- 10 files changed, 387 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 75cf6e8..9d3174e 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper now accepts only versioned, module-hashed allowlisted requests through a local Unix socket; the first executable action is read-only Linux `preflight`. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket; the first executable action is read-only Linux `preflight`. Its typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/docs/roadmap.md b/docs/roadmap.md index a994bb5..6fbff72 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -119,7 +119,7 @@ - [x] restricted root helper через Unix socket (первый allowlisted action `preflight`); - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; -- [ ] desired/factual configuration и drift (типизированные factual preflight facts уже сохраняются); +- [ ] desired/factual configuration и drift (`preflight` завершён; остальные action type ещё не подключены); - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). ## Этап 9 — базовая настройка и пользователи diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index 833ccbb..c390a42 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 6; + private const int CurrentSchemaVersion = 7; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -251,6 +251,28 @@ public sealed partial class ControlStore(IOptions options) await addNodePreflightFacts.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 7) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var addPreflightDesiredState = connection.CreateCommand(); + addPreflightDesiredState.Transaction = migration; + addPreflightDesiredState.CommandText = """ + CREATE TABLE IF NOT EXISTS node_preflight_desired_state ( + node_id TEXT PRIMARY KEY REFERENCES agents(node_id) ON DELETE CASCADE, + schema_version INTEGER NOT NULL, + desired_json TEXT NOT NULL, + version INTEGER NOT NULL, + updated_by TEXT NOT NULL, + audit_reason TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + PRAGMA user_version = 7; + """; + await addPreflightDesiredState.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 0b7135a..aea5554 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -479,6 +479,52 @@ control.MapGet("/agents/{nodeId}/facts/preflight", async ( var facts = await controlStore.GetPreflightFactsAsync(nodeId, cancellationToken); return facts is null ? Results.NotFound() : Results.Ok(facts); }); +control.MapPut("/agents/{nodeId}/desired/preflight", async ( + string nodeId, + PreflightDesiredStateUpdateRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!NodeIdValidator.IsValid(nodeId) || !PreflightDesiredStateValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["preflightDesiredState"] = + ["Invalid schema, requirements, architectures, audit reason, or idempotency key."] + }); + } + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var desired = await controlStore.SetPreflightDesiredStateAsync( + nodeId, request, actor, cancellationToken); + return Results.Ok(desired); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningNodeNotFoundException) + { + return Results.NotFound(); + } +}); +control.MapGet("/agents/{nodeId}/drift/preflight", async ( + string nodeId, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!NodeIdValidator.IsValid(nodeId)) + { + return Results.ValidationProblem(new Dictionary + { + ["nodeId"] = ["Invalid node id."] + }); + } + var assessment = await controlStore.AssessPreflightDriftAsync(nodeId, cancellationToken); + return assessment is null ? Results.NotFound() : Results.Ok(assessment); +}); control.MapPost("/agents/{nodeId}/provisioning/jobs", async ( string nodeId, ProvisioningJobCreateRequest request, @@ -955,3 +1001,19 @@ internal static class ProvisioningJobValidator && value.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-' or '.'); } + +internal static class PreflightDesiredStateValidator +{ + private static readonly HashSet SupportedArchitectures = + new(["x64", "x86", "arm", "arm64"], StringComparer.Ordinal); + + public static bool IsValid(PreflightDesiredStateUpdateRequest request) + => request.SchemaVersion == 1 + && request.Desired is not null + && request.Desired.AllowedArchitectures is { Length: >= 1 and <= 4 } + && request.Desired.AllowedArchitectures.Distinct(StringComparer.Ordinal).Count() + == request.Desired.AllowedArchitectures.Length + && request.Desired.AllowedArchitectures.All(SupportedArchitectures.Contains) + && request.AuditReason is { Length: >= 1 and <= 256 } + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); +} diff --git a/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs index 63ae71e..94d02b4 100644 --- a/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs @@ -97,4 +97,176 @@ public sealed partial class ControlStore nodeId, reader.GetInt32(0), facts, DateTimeOffset.Parse(reader.GetString(2)), reader.GetString(3), DateTimeOffset.Parse(reader.GetString(4))); } + + public async Task SetPreflightDesiredStateAsync( + string nodeId, + PreflightDesiredStateUpdateRequest request, + string actor, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"preflight-desired:{actor}:{nodeId}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.PreflightDesiredStateUpdateRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.NodePreflightDesiredState, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var exists = connection.CreateCommand(); + exists.Transaction = transaction; + exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);"; + exists.Parameters.AddWithValue("$node", nodeId); + if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1) + { + throw new ProvisioningNodeNotFoundException(nodeId); + } + + var readVersion = connection.CreateCommand(); + readVersion.Transaction = transaction; + readVersion.CommandText = + "SELECT version FROM node_preflight_desired_state WHERE node_id = $node;"; + readVersion.Parameters.AddWithValue("$node", nodeId); + var existingVersion = await readVersion.ExecuteScalarAsync(cancellationToken); + var now = DateTimeOffset.UtcNow; + var desired = new NodePreflightDesiredState( + nodeId, request.SchemaVersion, request.Desired, + existingVersion is null ? 1 : Convert.ToInt64(existingVersion) + 1, + actor, request.AuditReason, now); + var upsert = connection.CreateCommand(); + upsert.Transaction = transaction; + upsert.CommandText = """ + INSERT INTO node_preflight_desired_state( + node_id, schema_version, desired_json, version, + updated_by, audit_reason, updated_at) + VALUES ($node, $schema, $desired, $version, $actor, $reason, $updated) + ON CONFLICT(node_id) DO UPDATE SET + schema_version = excluded.schema_version, + desired_json = excluded.desired_json, + version = excluded.version, + updated_by = excluded.updated_by, + audit_reason = excluded.audit_reason, + updated_at = excluded.updated_at; + """; + upsert.Parameters.AddWithValue("$node", desired.NodeId); + upsert.Parameters.AddWithValue("$schema", desired.SchemaVersion); + upsert.Parameters.AddWithValue( + "$desired", + JsonSerializer.Serialize(desired.Desired, SmmJsonContext.Default.PreflightDesiredRequirements)); + upsert.Parameters.AddWithValue("$version", desired.Version); + upsert.Parameters.AddWithValue("$actor", desired.UpdatedBy); + upsert.Parameters.AddWithValue("$reason", desired.AuditReason); + upsert.Parameters.AddWithValue("$updated", desired.UpdatedAt.ToString("O")); + await upsert.ExecuteNonQueryAsync(cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, desired, + SmmJsonContext.Default.NodePreflightDesiredState, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, "provisioning.preflight.desired", nodeId, + JsonSerializer.Serialize(new { desired.SchemaVersion, desired.Version }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return desired; + } + + public async Task GetPreflightDesiredStateAsync( + string nodeId, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var command = connection.CreateCommand(); + command.CommandText = """ + SELECT schema_version, desired_json, version, updated_by, audit_reason, updated_at + FROM node_preflight_desired_state WHERE node_id = $node; + """; + command.Parameters.AddWithValue("$node", nodeId); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + var desired = JsonSerializer.Deserialize( + reader.GetString(1), SmmJsonContext.Default.PreflightDesiredRequirements) + ?? throw new InvalidDataException("Stored preflight desired state is invalid."); + return new NodePreflightDesiredState( + nodeId, reader.GetInt32(0), desired, reader.GetInt64(2), reader.GetString(3), + reader.GetString(4), DateTimeOffset.Parse(reader.GetString(5))); + } + + public async Task AssessPreflightDriftAsync( + string nodeId, + CancellationToken cancellationToken = default) + { + await using (var connection = await OpenAsync(cancellationToken)) + { + var exists = connection.CreateCommand(); + exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);"; + exists.Parameters.AddWithValue("$node", nodeId); + if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1) + { + return null; + } + } + var desired = await GetPreflightDesiredStateAsync(nodeId, cancellationToken); + var facts = await GetPreflightFactsAsync(nodeId, cancellationToken); + return PreflightDriftEvaluator.Assess(nodeId, desired, facts); + } +} + +internal static class PreflightDriftEvaluator +{ + public static PreflightDriftAssessment Assess( + string nodeId, + NodePreflightDesiredState? desired, + NodePreflightFacts? facts) + { + if (desired is null) + { + return new PreflightDriftAssessment( + nodeId, PreflightDriftStatuses.NotConfigured, [], null, facts); + } + if (facts is null) + { + return new PreflightDriftAssessment( + nodeId, PreflightDriftStatuses.Unknown, [PreflightDriftCodes.FactsMissing], desired, null); + } + + var drift = new List(); + AddMissing( + drift, desired.Desired.RequireSystemd, facts.Facts.HasSystemd, + PreflightDriftCodes.SystemdMissing); + AddMissing( + drift, desired.Desired.RequireSshd, facts.Facts.HasSshd, + PreflightDriftCodes.SshdMissing); + AddMissing( + drift, desired.Desired.RequireNftables, facts.Facts.HasNftables, + PreflightDriftCodes.NftablesMissing); + AddMissing( + drift, desired.Desired.RequireWireGuard, facts.Facts.HasWireGuard, + PreflightDriftCodes.WireGuardMissing); + AddMissing( + drift, desired.Desired.RequireApt, facts.Facts.HasApt, + PreflightDriftCodes.AptMissing); + if (!desired.Desired.AllowedArchitectures.Contains( + facts.Facts.Architecture, StringComparer.Ordinal)) + { + drift.Add(PreflightDriftCodes.ArchitectureUnsupported); + } + return new PreflightDriftAssessment( + nodeId, + drift.Count == 0 ? PreflightDriftStatuses.InSync : PreflightDriftStatuses.Drifted, + [.. drift], desired, facts); + } + + private static void AddMissing(List drift, bool required, bool present, string code) + { + if (required && !present) + { + drift.Add(code); + } + } } diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 1370bed..032735d 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -204,6 +204,57 @@ public sealed record NodePreflightFacts( string SourceJobId, DateTimeOffset UpdatedAt); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record PreflightDesiredRequirements( + bool RequireSystemd, + bool RequireSshd, + bool RequireNftables, + bool RequireWireGuard, + bool RequireApt, + string[] AllowedArchitectures); + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record PreflightDesiredStateUpdateRequest( + int SchemaVersion, + PreflightDesiredRequirements Desired, + string AuditReason, + string IdempotencyKey); + +public sealed record NodePreflightDesiredState( + string NodeId, + int SchemaVersion, + PreflightDesiredRequirements Desired, + long Version, + string UpdatedBy, + string AuditReason, + DateTimeOffset UpdatedAt); + +public sealed record PreflightDriftAssessment( + string NodeId, + string Status, + string[] DriftCodes, + NodePreflightDesiredState? Desired, + NodePreflightFacts? Facts); + +public static class PreflightDriftStatuses +{ + public const string NotConfigured = "NotConfigured"; + public const string Unknown = "Unknown"; + public const string InSync = "InSync"; + public const string Drifted = "Drifted"; +} + +public static class PreflightDriftCodes +{ + public const string FactsMissing = "facts.missing"; + public const string SystemdMissing = "systemd.missing"; + public const string SshdMissing = "sshd.missing"; + public const string NftablesMissing = "nftables.missing"; + public const string WireGuardMissing = "wireguard.missing"; + public const string AptMissing = "apt.missing"; + public const string ArchitectureUnsupported = "architecture.unsupported"; +} + public sealed record ProvisioningJob( string Id, string NodeId, diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 873e40e..524d64c 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -33,6 +33,10 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningPreflightResult))] [JsonSerializable(typeof(ProvisioningPreflightReportRequest))] [JsonSerializable(typeof(NodePreflightFacts))] +[JsonSerializable(typeof(PreflightDesiredRequirements))] +[JsonSerializable(typeof(PreflightDesiredStateUpdateRequest))] +[JsonSerializable(typeof(NodePreflightDesiredState))] +[JsonSerializable(typeof(PreflightDriftAssessment))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index be7aec1..1fef077 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -216,6 +216,77 @@ public sealed class ControlApiTests : IAsyncDisposable .ReadFromJsonAsync(cancellationToken); Assert.Equal(created.Id, stored!.SourceJobId); + var desiredRequest = new + { + schemaVersion = 1, + desired = new + { + requireSystemd = true, + requireSshd = true, + requireNftables = true, + requireWireGuard = true, + requireApt = true, + allowedArchitectures = new[] { "x64", "arm64" } + }, + auditReason = "Detect missing host capabilities", + idempotencyKey = Guid.NewGuid().ToString() + }; + using var desiredResponse = await operatorClient.PutAsJsonAsync( + $"/api/v1/control/agents/{nodeId}/desired/preflight", + desiredRequest, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, desiredResponse.StatusCode); + var desired = await desiredResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal(1, desired!.Version); + using var desiredReplayResponse = await operatorClient.PutAsJsonAsync( + $"/api/v1/control/agents/{nodeId}/desired/preflight", + desiredRequest, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, desiredReplayResponse.StatusCode); + var desiredReplay = await desiredReplayResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal(desired.UpdatedAt, desiredReplay!.UpdatedAt); + + using var driftResponse = await operatorClient.GetAsync( + $"/api/v1/control/agents/{nodeId}/drift/preflight", cancellationToken); + Assert.Equal(HttpStatusCode.OK, driftResponse.StatusCode); + var drift = await driftResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal("Drifted", drift!.Status); + Assert.Equal(["wireguard.missing"], drift.DriftCodes); + + var synchronizedDesiredRequest = new + { + schemaVersion = 1, + desired = new + { + requireSystemd = true, + requireSshd = true, + requireNftables = true, + requireWireGuard = false, + requireApt = true, + allowedArchitectures = new[] { "x64", "arm64" } + }, + auditReason = "Accept host without WireGuard tooling", + idempotencyKey = Guid.NewGuid().ToString() + }; + using var synchronizedDesiredResponse = await operatorClient.PutAsJsonAsync( + $"/api/v1/control/agents/{nodeId}/desired/preflight", + synchronizedDesiredRequest, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, synchronizedDesiredResponse.StatusCode); + var synchronizedDesired = await synchronizedDesiredResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal(2, synchronizedDesired!.Version); + using var synchronizedDriftResponse = await operatorClient.GetAsync( + $"/api/v1/control/agents/{nodeId}/drift/preflight", cancellationToken); + Assert.Equal(HttpStatusCode.OK, synchronizedDriftResponse.StatusCode); + var synchronizedDrift = await synchronizedDriftResponse.Content + .ReadFromJsonAsync(cancellationToken); + Assert.Equal("InSync", synchronizedDrift!.Status); + Assert.Empty(synchronizedDrift.DriftCodes); + var progress = new { state = "Running", diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 528d5e0..3954266 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(6L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(7L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index faf3efa..cfc47bc 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(6L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(7L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); -- 2.45.2 From 16885258f0238d218cb147a497d52db1030116e5 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 23:49:51 +0700 Subject: [PATCH 27/31] Define strict base install schema catalog --- docs/roadmap.md | 2 +- src/ServerMonitorManager.Control/Program.cs | 67 ++++++++++++++++--- src/ServerMonitorManager.Core/Contracts.cs | 39 +++++++++++ .../SmmJsonContext.cs | 5 ++ .../ControlApiTests.cs | 46 ++++++++++++- .../ProvisioningHelperTests.cs | 14 ++++ 6 files changed, 163 insertions(+), 10 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 6fbff72..45b8a71 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -126,7 +126,7 @@ - [ ] preflight ОС, архитектуры, SSH, firewall, APT и capabilities; - [ ] Desktop wizard timezone/locale/packages/swap/unattended upgrades; -- [ ] versioned package allowlist; +- [x] versioned package allowlist (catalog v1 с фиксированными package groups); - [ ] root-only backups и symlink protection; - [ ] user lifecycle без sudo по умолчанию; - [ ] SSH public keys, fingerprints и permissions; diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index aea5554..e608ea4 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -464,6 +464,8 @@ agents.MapPost("/provisioning/jobs/{id}/preflight-facts", async ( var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray())); +control.MapGet("/provisioning/catalogs/system-base-install/1", () => + Results.Ok(SystemBaseInstallCatalogDefinition.Create())); control.MapGet("/agents/{nodeId}/facts/preflight", async ( string nodeId, ControlStore controlStore, @@ -945,14 +947,35 @@ internal static class ProvisioningJobValidator private const int MaximumParametersBytes = 16 * 1024; public static bool IsValid(ProvisioningJobCreateRequest request) - => request.SchemaVersion == 1 - && request.ActionType is "preflight" or "system.base-install" - && request.Parameters.ValueKind == JsonValueKind.Object - && !request.Parameters.EnumerateObject().Any() - && request.Parameters.GetRawText().Length <= MaximumParametersBytes - && request.TtlMinutes is >= 5 and <= 1440 - && request.AuditReason.Length is >= 1 and <= 256 - && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + { + if (request.SchemaVersion != 1 + || request.Parameters.ValueKind != JsonValueKind.Object + || request.Parameters.GetRawText().Length > MaximumParametersBytes + || request.TtlMinutes is < 5 or > 1440 + || request.AuditReason is not { Length: >= 1 and <= 256 } + || !IdempotencyKeyValidator.IsValid(request.IdempotencyKey)) + { + return false; + } + if (request.ActionType == "preflight") + { + return !request.Parameters.EnumerateObject().Any(); + } + if (request.ActionType != "system.base-install") + { + return false; + } + try + { + var parameters = JsonSerializer.Deserialize( + request.Parameters, SmmJsonContext.Default.SystemBaseInstallParameters); + return parameters is not null && IsValid(parameters); + } + catch (JsonException) + { + return false; + } + } public static bool IsValid(ProvisioningJobCommandRequest request) => request.Reason.Length is >= 1 and <= 256 @@ -989,6 +1012,22 @@ internal static class ProvisioningJobValidator public static bool RequiresConfirmation(string actionType) => actionType == "system.base-install"; + private static bool IsValid(SystemBaseInstallParameters parameters) + => IsSafeTimezone(parameters.Timezone) + && IsSafeLocale(parameters.Locale) + && (!parameters.AptUpgrade || parameters.AptUpdate) + && parameters.PackageCatalogVersion == SystemBaseInstallCatalogDefinition.Version + && parameters.PackageGroupIds is { Length: <= 4 } + && parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count() + == parameters.PackageGroupIds.Length + && parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup) + && parameters.SwapMode is "disabled" or "automatic" or "explicit" + && (parameters.SwapMode == "explicit" + ? parameters.SwapSizeMiB is >= 128 and <= 1_048_576 + : parameters.SwapSizeMiB is null) + && parameters.VmSwappiness is >= 0 and <= 200 + && parameters.RebootPolicy == "never"; + private static bool IsSafeCode(string value, int maximumLength) => value.Length is >= 1 && value.Length <= maximumLength && value.All(character => character is >= 'a' and <= 'z' @@ -1000,6 +1039,18 @@ internal static class ProvisioningJobValidator && value.Length is >= 1 && value.Length <= maximumLength && value.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-' or '.'); + + private static bool IsSafeTimezone(string? value) + => value is { Length: >= 1 and <= 64 } + && value[0] is not '/' and not '.' + && !value.Contains("..", StringComparison.Ordinal) + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '/' or '_' or '-' or '+'); + + private static bool IsSafeLocale(string? value) + => value is { Length: >= 1 and <= 32 } + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '_' or '-' or '.' or '@'); } internal static class PreflightDesiredStateValidator diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 032735d..512cab3 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -236,6 +236,28 @@ public sealed record PreflightDriftAssessment( NodePreflightDesiredState? Desired, NodePreflightFacts? Facts); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record SystemBaseInstallParameters( + string Timezone, + string Locale, + bool AptUpdate, + bool AptUpgrade, + int PackageCatalogVersion, + string[] PackageGroupIds, + string SwapMode, + int? SwapSizeMiB, + int VmSwappiness, + bool EnableUnattendedUpgrades, + string RebootPolicy); + +public sealed record SystemPackageGroup( + string Id, + string[] Packages); + +public sealed record SystemBaseInstallCatalog( + int Version, + SystemPackageGroup[] Groups); + public static class PreflightDriftStatuses { public const string NotConfigured = "NotConfigured"; @@ -306,3 +328,20 @@ public static class ProvisioningActionCatalog public const string PreflightModuleHash = "2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b"; } + +public static class SystemBaseInstallCatalogDefinition +{ + public const int Version = 1; + + public static SystemBaseInstallCatalog Create() + => new(Version, + [ + new("core", ["ca-certificates", "curl", "jq"]), + new("development", ["build-essential", "git"]), + new("diagnostics", ["htop", "iotop"]), + new("container-host", ["dbus-user-session", "uidmap"]) + ]); + + public static bool ContainsGroup(string id) + => id is "core" or "development" or "diagnostics" or "container-host"; +} diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 524d64c..255898e 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; namespace ServerMonitorManager.Core; +[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] [JsonSerializable(typeof(EnrollmentRequest))] [JsonSerializable(typeof(EnrollmentResponse))] [JsonSerializable(typeof(AgentHeartbeat))] @@ -37,6 +38,10 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(PreflightDesiredStateUpdateRequest))] [JsonSerializable(typeof(NodePreflightDesiredState))] [JsonSerializable(typeof(PreflightDriftAssessment))] +[JsonSerializable(typeof(SystemBaseInstallParameters))] +[JsonSerializable(typeof(SystemPackageGroup))] +[JsonSerializable(typeof(SystemPackageGroup[]))] +[JsonSerializable(typeof(SystemBaseInstallCatalog))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 1fef077..ca4e601 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -96,13 +96,32 @@ public sealed class ControlApiTests : IAsyncDisposable using var client = _factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc"); client.DefaultRequestHeaders.Add("X-Test-Role", "Operator"); + var catalog = await client.GetFromJsonAsync( + "/api/v1/control/provisioning/catalogs/system-base-install/1", + cancellationToken); + Assert.Equal(1, catalog!.Version); + Assert.Contains(catalog.Groups, group => group.Id == "development" + && group.Packages.Contains("git")); using var response = await client.PostAsJsonAsync( "/api/v1/control/agents/home/provisioning/jobs", new { actionType = "system.base-install", schemaVersion = 1, - parameters = new { }, + parameters = new + { + timezone = "UTC", + locale = "en_US.UTF-8", + aptUpdate = true, + aptUpgrade = false, + packageCatalogVersion = 1, + packageGroupIds = new[] { "core", "development" }, + swapMode = "automatic", + swapSizeMiB = (int?)null, + vmSwappiness = 60, + enableUnattendedUpgrades = true, + rebootPolicy = "never" + }, ttlMinutes = 60, auditReason = "API integration test", idempotencyKey = Guid.NewGuid().ToString() @@ -114,6 +133,31 @@ public sealed class ControlApiTests : IAsyncDisposable cancellationToken); Assert.NotNull(job); Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.AwaitingConfirmation, job.State); + Assert.Equal(HttpStatusCode.BadRequest, (await client.PostAsJsonAsync( + "/api/v1/control/agents/home/provisioning/jobs", + new + { + actionType = "system.base-install", + schemaVersion = 1, + parameters = new + { + timezone = "UTC", + locale = "en_US.UTF-8", + aptUpdate = true, + aptUpgrade = false, + packageCatalogVersion = 1, + packageGroupIds = new[] { "curl" }, + swapMode = "disabled", + swapSizeMiB = (int?)null, + vmSwappiness = 60, + enableUnattendedUpgrades = false, + rebootPolicy = "never" + }, + ttlMinutes = 60, + auditReason = "Reject arbitrary package input", + idempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken)).StatusCode); Assert.Equal(HttpStatusCode.OK, (await client.GetAsync( $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode); using var eventsResponse = await client.GetAsync( diff --git a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs index f5c8190..ca33b0d 100644 --- a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs @@ -53,4 +53,18 @@ public sealed class ProvisioningHelperTests Assert.Throws(() => JsonSerializer.Deserialize(json, SmmJsonContext.Default.ProvisioningHelperRequest)); } + + [Fact] + public void BaseInstallSchemaRejectsCommandText() + { + const string json = """ + {"timezone":"UTC","locale":"en_US.UTF-8","aptUpdate":true,"aptUpgrade":false, + "packageCatalogVersion":1,"packageGroupIds":["core"],"swapMode":"disabled", + "swapSizeMiB":null,"vmSwappiness":60,"enableUnattendedUpgrades":true, + "rebootPolicy":"never","command":"id"} + """; + + Assert.Throws(() => + JsonSerializer.Deserialize(json, SmmJsonContext.Default.SystemBaseInstallParameters)); + } } -- 2.45.2 From 7be4f153fb5d66387a0cbd21d178c8ccf5e90c28 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 23:59:51 +0700 Subject: [PATCH 28/31] Add safe base install plan generation --- README.md | 2 +- docs/roadmap.md | 2 +- .../ProvisioningHelperClient.cs | 33 +++++++-- src/ServerMonitorManager.Control/Program.cs | 38 +--------- src/ServerMonitorManager.Core/Contracts.cs | 74 ++++++++++++++++++- .../SmmJsonContext.cs | 1 + .../ProvisioningHelperServer.cs | 57 ++++++++++++-- .../ProvisioningHelperTests.cs | 21 ++++++ 8 files changed, 178 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 9d3174e..8429a96 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket; the first executable action is read-only Linux `preflight`. Its typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and can build a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/docs/roadmap.md b/docs/roadmap.md index 45b8a71..c6fc5ff 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -116,7 +116,7 @@ - [x] атомарный Agent job channel только для собственного `node_id`; - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; -- [x] restricted root helper через Unix socket (первый allowlisted action `preflight`); +- [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`); - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift (`preflight` завершён; остальные action type ещё не подключены); diff --git a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs index 48b13a8..d0c2673 100644 --- a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs +++ b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs @@ -16,6 +16,33 @@ public sealed class ProvisioningHelperClient(string socketPath) var request = new ProvisioningHelperRequest( "1", job.Id, job.ActionType, job.SchemaVersion, ProvisioningActionCatalog.PreflightModuleHash, job.Parameters); + var response = await SendAsync(request, cancellationToken); + if (!response.Success || response.Preflight is null) + { + throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}"); + } + return response.Preflight; + } + + public async Task CreateBaseInstallPlanAsync( + ProvisioningJob job, + CancellationToken cancellationToken) + { + var request = new ProvisioningHelperRequest( + "1", job.Id, job.ActionType, job.SchemaVersion, + ProvisioningActionCatalog.SystemBaseInstallModuleHash, job.Parameters); + var response = await SendAsync(request, cancellationToken); + if (!response.Success || response.BaseInstallPlan is null) + { + throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}"); + } + return response.BaseInstallPlan; + } + + private async Task SendAsync( + ProvisioningHelperRequest request, + CancellationToken cancellationToken) + { using var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); await socket.ConnectAsync(new UnixDomainSocketEndPoint(socketPath), cancellationToken); await using var stream = new NetworkStream(socket, ownsSocket: false); @@ -24,11 +51,7 @@ public sealed class ProvisioningHelperClient(string socketPath) var payload = await ReadResponseAsync(stream, cancellationToken); var response = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperResponse) ?? throw new InvalidDataException("Provisioning helper returned an empty response."); - if (!response.Success || response.Preflight is null) - { - throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}"); - } - return response.Preflight; + return response; } private static async Task ReadResponseAsync(Stream stream, CancellationToken cancellationToken) diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index e608ea4..5988f7f 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -965,16 +965,7 @@ internal static class ProvisioningJobValidator { return false; } - try - { - var parameters = JsonSerializer.Deserialize( - request.Parameters, SmmJsonContext.Default.SystemBaseInstallParameters); - return parameters is not null && IsValid(parameters); - } - catch (JsonException) - { - return false; - } + return SystemBaseInstallSchema.TryParse(request.Parameters, out _); } public static bool IsValid(ProvisioningJobCommandRequest request) @@ -1012,22 +1003,6 @@ internal static class ProvisioningJobValidator public static bool RequiresConfirmation(string actionType) => actionType == "system.base-install"; - private static bool IsValid(SystemBaseInstallParameters parameters) - => IsSafeTimezone(parameters.Timezone) - && IsSafeLocale(parameters.Locale) - && (!parameters.AptUpgrade || parameters.AptUpdate) - && parameters.PackageCatalogVersion == SystemBaseInstallCatalogDefinition.Version - && parameters.PackageGroupIds is { Length: <= 4 } - && parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count() - == parameters.PackageGroupIds.Length - && parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup) - && parameters.SwapMode is "disabled" or "automatic" or "explicit" - && (parameters.SwapMode == "explicit" - ? parameters.SwapSizeMiB is >= 128 and <= 1_048_576 - : parameters.SwapSizeMiB is null) - && parameters.VmSwappiness is >= 0 and <= 200 - && parameters.RebootPolicy == "never"; - private static bool IsSafeCode(string value, int maximumLength) => value.Length is >= 1 && value.Length <= maximumLength && value.All(character => character is >= 'a' and <= 'z' @@ -1040,17 +1015,6 @@ internal static class ProvisioningJobValidator && value.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-' or '.'); - private static bool IsSafeTimezone(string? value) - => value is { Length: >= 1 and <= 64 } - && value[0] is not '/' and not '.' - && !value.Contains("..", StringComparison.Ordinal) - && value.All(character => char.IsAsciiLetterOrDigit(character) - || character is '/' or '_' or '-' or '+'); - - private static bool IsSafeLocale(string? value) - => value is { Length: >= 1 and <= 32 } - && value.All(character => char.IsAsciiLetterOrDigit(character) - || character is '_' or '-' or '.' or '@'); } internal static class PreflightDesiredStateValidator diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 512cab3..a9f5a53 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -177,7 +177,8 @@ public sealed record ProvisioningHelperResponse( bool Success, string Code, string Message, - ProvisioningPreflightResult? Preflight); + ProvisioningPreflightResult? Preflight, + SystemBaseInstallPlan? BaseInstallPlan); [JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] public sealed record ProvisioningPreflightResult( @@ -258,6 +259,19 @@ public sealed record SystemBaseInstallCatalog( int Version, SystemPackageGroup[] Groups); +public sealed record SystemBaseInstallPlan( + string Timezone, + string Locale, + bool AptUpdate, + bool AptUpgrade, + string[] Packages, + string SwapMode, + int? SwapSizeMiB, + int VmSwappiness, + bool EnableUnattendedUpgrades, + string RebootPolicy, + string[] Warnings); + public static class PreflightDriftStatuses { public const string NotConfigured = "NotConfigured"; @@ -327,6 +341,8 @@ public static class ProvisioningActionCatalog { public const string PreflightModuleHash = "2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b"; + public const string SystemBaseInstallModuleHash = + "355d55e214b941160a32957ced1a681e3c7324f94ecb340f26042f0c3b59b99e"; } public static class SystemBaseInstallCatalogDefinition @@ -344,4 +360,60 @@ public static class SystemBaseInstallCatalogDefinition public static bool ContainsGroup(string id) => id is "core" or "development" or "diagnostics" or "container-host"; + + public static string[] ExpandGroups(IEnumerable ids) + { + var selected = ids.ToHashSet(StringComparer.Ordinal); + return Create().Groups + .Where(group => selected.Contains(group.Id)) + .SelectMany(group => group.Packages) + .Distinct(StringComparer.Ordinal) + .ToArray(); + } +} + +public static class SystemBaseInstallSchema +{ + public static bool TryParse(JsonElement json, out SystemBaseInstallParameters? parameters) + { + try + { + parameters = JsonSerializer.Deserialize( + json, SmmJsonContext.Default.SystemBaseInstallParameters); + return parameters is not null && IsValid(parameters); + } + catch (JsonException) + { + parameters = null; + return false; + } + } + + public static bool IsValid(SystemBaseInstallParameters parameters) + => IsSafeTimezone(parameters.Timezone) + && IsSafeLocale(parameters.Locale) + && (!parameters.AptUpgrade || parameters.AptUpdate) + && parameters.PackageCatalogVersion == SystemBaseInstallCatalogDefinition.Version + && parameters.PackageGroupIds is { Length: <= 4 } + && parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count() + == parameters.PackageGroupIds.Length + && parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup) + && parameters.SwapMode is "disabled" or "automatic" or "explicit" + && (parameters.SwapMode == "explicit" + ? parameters.SwapSizeMiB is >= 128 and <= 1_048_576 + : parameters.SwapSizeMiB is null) + && parameters.VmSwappiness is >= 0 and <= 200 + && parameters.RebootPolicy == "never"; + + private static bool IsSafeTimezone(string? value) + => value is { Length: >= 1 and <= 64 } + && value[0] is not '/' and not '.' + && !value.Contains("..", StringComparison.Ordinal) + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '/' or '_' or '-' or '+'); + + private static bool IsSafeLocale(string? value) + => value is { Length: >= 1 and <= 32 } + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '_' or '-' or '.' or '@'); } diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 255898e..a77cce2 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -42,6 +42,7 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(SystemPackageGroup))] [JsonSerializable(typeof(SystemPackageGroup[]))] [JsonSerializable(typeof(SystemBaseInstallCatalog))] +[JsonSerializable(typeof(SystemBaseInstallPlan))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs index 55b89b5..b1c0455 100644 --- a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs +++ b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs @@ -46,9 +46,22 @@ public sealed class ProvisioningHelperServer(string socketPath) { return Failure("request.invalid-job", "Invalid provisioning job identifier."); } - if (request.ActionType != "preflight" || request.SchemaVersion != 1 - || request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash - || request.Parameters.ValueKind != JsonValueKind.Object + if (request.SchemaVersion != 1 || request.Parameters.ValueKind != JsonValueKind.Object) + { + return Failure("action.denied", "The requested action is not allowed."); + } + + return request.ActionType switch + { + "preflight" => ExecutePreflight(request), + "system.base-install" => CreateBaseInstallPlan(request), + _ => Failure("action.denied", "The requested action is not allowed.") + }; + } + + private static ProvisioningHelperResponse ExecutePreflight(ProvisioningHelperRequest request) + { + if (request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash || request.Parameters.EnumerateObject().Any()) { return Failure("action.denied", "The requested action is not allowed."); @@ -64,7 +77,41 @@ public sealed class ProvisioningHelperServer(string socketPath) Exists("/usr/sbin/nft", "/usr/bin/nft", "/sbin/nft"), Exists("/usr/bin/wg", "/usr/sbin/wg", "/bin/wg"), Exists("/usr/bin/apt-get", "/bin/apt-get")); - return new ProvisioningHelperResponse(true, "preflight.completed", "Preflight completed.", result); + return new ProvisioningHelperResponse( + true, "preflight.completed", "Preflight completed.", result, null); + } + + private static ProvisioningHelperResponse CreateBaseInstallPlan(ProvisioningHelperRequest request) + { + if (request.ModuleHash != ProvisioningActionCatalog.SystemBaseInstallModuleHash + || !SystemBaseInstallSchema.TryParse(request.Parameters, out var parameters)) + { + return Failure("action.denied", "The requested action is not allowed."); + } + + var warnings = new List(); + if (!Exists("/usr/bin/apt-get", "/bin/apt-get")) + { + warnings.Add("apt.missing"); + } + if (!File.Exists(Path.Combine("/usr/share/zoneinfo", parameters!.Timezone))) + { + warnings.Add("timezone.missing"); + } + var plan = new SystemBaseInstallPlan( + parameters.Timezone, + parameters.Locale, + parameters.AptUpdate, + parameters.AptUpgrade, + SystemBaseInstallCatalogDefinition.ExpandGroups(parameters.PackageGroupIds), + parameters.SwapMode, + parameters.SwapSizeMiB, + parameters.VmSwappiness, + parameters.EnableUnattendedUpgrades, + parameters.RebootPolicy, + [.. warnings]); + return new ProvisioningHelperResponse( + true, "system.base-install.plan-ready", "Base install plan is ready.", null, plan); } private static async Task HandleAsync(Socket socket, CancellationToken cancellationToken) @@ -135,5 +182,5 @@ public sealed class ProvisioningHelperServer(string socketPath) private static bool Exists(params string[] paths) => paths.Any(File.Exists); private static ProvisioningHelperResponse Failure(string code, string message) - => new(false, code, message, null); + => new(false, code, message, null, null); } diff --git a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs index ca33b0d..2ed11c2 100644 --- a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs @@ -67,4 +67,25 @@ public sealed class ProvisioningHelperTests Assert.Throws(() => JsonSerializer.Deserialize(json, SmmJsonContext.Default.SystemBaseInstallParameters)); } + + [Fact] + public void HelperBuildsDeterministicBaseInstallPlanWithoutCommands() + { + var parameters = new SystemBaseInstallParameters( + "UTC", "en_US.UTF-8", true, false, 1, + ["development", "core"], "disabled", null, 60, true, "never"); + var json = JsonSerializer.SerializeToElement( + parameters, SmmJsonContext.Default.SystemBaseInstallParameters); + var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest( + "1", new string('c', 32), "system.base-install", 1, + ProvisioningActionCatalog.SystemBaseInstallModuleHash, json)); + + Assert.True(response.Success); + Assert.Equal("system.base-install.plan-ready", response.Code); + Assert.Null(response.Preflight); + Assert.Equal( + ["ca-certificates", "curl", "jq", "build-essential", "git"], + response.BaseInstallPlan!.Packages); + Assert.Equal("never", response.BaseInstallPlan.RebootPolicy); + } } -- 2.45.2 From 2111a0950349b9b56dae4c54c812e2d132ccc671 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Mon, 20 Jul 2026 00:19:51 +0700 Subject: [PATCH 29/31] Add pre-confirmation base install plans --- README.md | 2 +- docs/provisioning-vpn-requirements.md | 1 + docs/roadmap.md | 1 + src/ServerMonitorManager.Agent/AgentClient.cs | 47 +++++- .../ControlStore.cs | 21 ++- src/ServerMonitorManager.Control/Program.cs | 59 ++++++++ .../ProvisioningBaseInstallPlanStore.cs | 138 ++++++++++++++++++ .../ProvisioningStore.cs | 14 +- src/ServerMonitorManager.Core/Contracts.cs | 36 +++++ .../SmmJsonContext.cs | 2 + .../ControlApiTests.cs | 2 +- .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 54 ++++++- 13 files changed, 363 insertions(+), 16 deletions(-) create mode 100644 src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs diff --git a/README.md b/README.md index 8429a96..9907fbd 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and can build a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and builds a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Control independently validates and stores that immutable plan before the job enters `AwaitingConfirmation`; confirmed base-install jobs remain safely queued until the mutating executor and factual verification are implemented. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/docs/provisioning-vpn-requirements.md b/docs/provisioning-vpn-requirements.md index f53132f..71d15b3 100644 --- a/docs/provisioning-vpn-requirements.md +++ b/docs/provisioning-vpn-requirements.md @@ -278,6 +278,7 @@ GET /api/v1/nodes/{nodeId}/configuration POST /api/v1/nodes/{nodeId}/provisioning/preflight POST /api/v1/nodes/{nodeId}/provisioning/jobs GET /api/v1/provisioning/jobs/{jobId} +GET /api/v1/provisioning/jobs/{jobId}/plan POST /api/v1/provisioning/jobs/{jobId}/confirm POST /api/v1/provisioning/jobs/{jobId}/cancel POST /api/v1/provisioning/jobs/{jobId}/rollback diff --git a/docs/roadmap.md b/docs/roadmap.md index c6fc5ff..c547753 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -117,6 +117,7 @@ - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; - [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`); +- [x] двухфазный `system.base-install`: сохранённый проверенный plan до Operator confirmation; - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift (`preflight` завершён; остальные action type ещё не подключены); diff --git a/src/ServerMonitorManager.Agent/AgentClient.cs b/src/ServerMonitorManager.Agent/AgentClient.cs index 8d17822..0a0e1d2 100644 --- a/src/ServerMonitorManager.Agent/AgentClient.cs +++ b/src/ServerMonitorManager.Agent/AgentClient.cs @@ -136,7 +136,36 @@ internal sealed class AgentClient(AgentOptions options) cancellationToken) ?? throw new InvalidOperationException("Control service returned an empty provisioning job."); - if (job.ActionType != "preflight" || job.SchemaVersion != 1) + if (job.SchemaVersion != 1) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, + "dispatch", "action.unsupported", "Unsupported provisioning action.", cancellationToken); + return; + } + + if (job.ActionType == "system.base-install" + && job.State == ProvisioningJobStates.Preflight) + { + try + { + var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath); + var plan = await helper.CreateBaseInstallPlanAsync(job, cancellationToken); + await ReportBaseInstallPlanAsync(client, job, plan, cancellationToken); + Console.WriteLine($"Base installation plan {job.Id} is awaiting confirmation."); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, + "preflight", "base-install.plan-failed", + "Base installation plan generation failed.", cancellationToken); + Console.Error.WriteLine($"Base installation plan {job.Id} failed: {exception.Message}"); + } + return; + } + + if (job.ActionType != "preflight") { await ReportProvisioningAsync( client, job, ProvisioningJobStates.Failed, job.ProgressPercent, @@ -210,6 +239,22 @@ internal sealed class AgentClient(AgentOptions options) response.EnsureSuccessStatusCode(); } + private static async Task ReportBaseInstallPlanAsync( + HttpClient client, + ProvisioningJob job, + SystemBaseInstallPlan plan, + CancellationToken cancellationToken) + { + var request = new SystemBaseInstallPlanReportRequest( + plan, CreateOperationId(job.Id, "base-install-plan")); + using var response = await client.PostAsJsonAsync( + $"api/v1/agents/provisioning/jobs/{job.Id}/base-install-plan", + request, + SmmJsonContext.Default.SystemBaseInstallPlanReportRequest, + cancellationToken); + response.EnsureSuccessStatusCode(); + } + private static string CreateOperationId(string jobId, string state) { var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}")); diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index c390a42..469294a 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 7; + private const int CurrentSchemaVersion = 8; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -273,6 +273,25 @@ public sealed partial class ControlStore(IOptions options) await addPreflightDesiredState.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 8) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var addBaseInstallPlans = connection.CreateCommand(); + addBaseInstallPlans.Transaction = migration; + addBaseInstallPlans.CommandText = """ + CREATE TABLE IF NOT EXISTS provisioning_base_install_plans ( + job_id TEXT PRIMARY KEY REFERENCES provisioning_jobs(id) ON DELETE CASCADE, + schema_version INTEGER NOT NULL, + plan_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + PRAGMA user_version = 8; + """; + await addBaseInstallPlans.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 5988f7f..0a55761 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -460,6 +460,44 @@ agents.MapPost("/provisioning/jobs/{id}/preflight-facts", async ( return Results.Conflict(new ProblemDetails { Title = exception.Message }); } }); +agents.MapPost("/provisioning/jobs/{id}/base-install-plan", async ( + string id, + SystemBaseInstallPlanReportRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) + || !NodeIdValidator.IsValid(nodeId) + || !ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["baseInstallPlan"] = ["Invalid job id, plan, or idempotency key."] + }); + } + + try + { + var plan = await controlStore.RecordBaseInstallPlanAsync( + nodeId, id, request, cancellationToken); + return plan is null ? Results.NotFound() : Results.Ok(plan); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } + catch (ProvisioningPlanValidationException exception) + { + return Results.BadRequest(new ProblemDetails { Title = exception.Message }); + } +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => @@ -581,6 +619,21 @@ control.MapGet("/provisioning/jobs/{id}", async ( var job = await controlStore.GetProvisioningJobAsync(id, cancellationToken); return job is null ? Results.NotFound() : Results.Ok(job); }); +control.MapGet("/provisioning/jobs/{id}/plan", async ( + string id, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id)) + { + return Results.ValidationProblem(new Dictionary + { + ["baseInstallPlan"] = ["Invalid provisioning job id."] + }); + } + var plan = await controlStore.GetBaseInstallPlanAsync(id, cancellationToken); + return plan is null ? Results.NotFound() : Results.Ok(plan); +}); control.MapGet("/provisioning/jobs/{id}/events", async ( string id, int? limit, @@ -997,6 +1050,12 @@ internal static class ProvisioningJobValidator && request.Facts.Architecture is "x64" or "x86" or "arm" or "arm64" && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValid(SystemBaseInstallPlanReportRequest request) + => request.Plan is not null + && request.Plan.Packages is { Length: <= 64 } + && request.Plan.Warnings is { Length: <= 2 } + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValidId(string id) => id.Length == 32 && Guid.TryParseExact(id, "N", out _); diff --git a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs new file mode 100644 index 0000000..fe93635 --- /dev/null +++ b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs @@ -0,0 +1,138 @@ +using System.Text.Json; +using Microsoft.Data.Sqlite; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Control; + +public sealed partial class ControlStore +{ + public async Task RecordBaseInstallPlanAsync( + string nodeId, + string jobId, + SystemBaseInstallPlanReportRequest request, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-base-install-plan:{nodeId}:{jobId}:{request.IdempotencyKey}"; + var requestHash = Fingerprint( + request, SmmJsonContext.Default.SystemBaseInstallPlanReportRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningBaseInstallPlanRecord, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken); + if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal)) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (job.ActionType != "system.base-install" || job.SchemaVersion != 1 + || job.State != ProvisioningJobStates.Preflight || !job.ConfirmationRequired + || job.ConfirmedAt is not null) + { + throw new ProvisioningTransitionException(job.State, ProvisioningJobStates.AwaitingConfirmation); + } + if (!SystemBaseInstallSchema.TryParse(job.Parameters, out var parameters) + || parameters is null + || !SystemBaseInstallSchema.IsValidPlan(parameters, request.Plan)) + { + throw new ProvisioningPlanValidationException(); + } + + var now = DateTimeOffset.UtcNow; + var record = new ProvisioningBaseInstallPlanRecord( + jobId, nodeId, job.SchemaVersion, request.Plan, now); + var insert = connection.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = """ + INSERT INTO provisioning_base_install_plans( + job_id, schema_version, plan_json, created_at) + VALUES ($job, $schema, $plan, $created); + """; + insert.Parameters.AddWithValue("$job", record.JobId); + insert.Parameters.AddWithValue("$schema", record.SchemaVersion); + insert.Parameters.AddWithValue( + "$plan", JsonSerializer.Serialize(record.Plan, SmmJsonContext.Default.SystemBaseInstallPlan)); + insert.Parameters.AddWithValue("$created", record.CreatedAt.ToString("O")); + await insert.ExecuteNonQueryAsync(cancellationToken); + + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, + progress_percent = 25, + current_step = 'awaiting-confirmation', + updated_at = $updated, + version = version + 1 + WHERE id = $id AND node_id = $node AND version = $version + AND state = $preflight AND confirmed_at IS NULL; + """; + update.Parameters.AddWithValue("$state", ProvisioningJobStates.AwaitingConfirmation); + update.Parameters.AddWithValue("$updated", now.ToString("O")); + update.Parameters.AddWithValue("$id", job.Id); + update.Parameters.AddWithValue("$node", nodeId); + update.Parameters.AddWithValue("$version", job.Version); + update.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(job.State, ProvisioningJobStates.AwaitingConfirmation); + } + + await WriteProvisioningEventAsync( + connection, transaction, jobId, "base-install.plan-recorded", + ProvisioningJobStates.AwaitingConfirmation, + "Validated base installation plan is awaiting operator confirmation.", now, + cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, record, + SmmJsonContext.Default.ProvisioningBaseInstallPlanRecord, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.base-install.plan", jobId, + JsonSerializer.Serialize(new + { + record.SchemaVersion, + PackageCount = record.Plan.Packages.Length, + WarningCodes = record.Plan.Warnings + }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return record; + } + + public async Task GetBaseInstallPlanAsync( + string jobId, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var command = connection.CreateCommand(); + command.CommandText = """ + SELECT p.schema_version, p.plan_json, p.created_at, j.node_id + FROM provisioning_base_install_plans p + INNER JOIN provisioning_jobs j ON j.id = p.job_id + WHERE p.job_id = $job; + """; + command.Parameters.AddWithValue("$job", jobId); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + var plan = JsonSerializer.Deserialize( + reader.GetString(1), SmmJsonContext.Default.SystemBaseInstallPlan) + ?? throw new InvalidDataException("Stored base installation plan is invalid."); + return new ProvisioningBaseInstallPlanRecord( + jobId, reader.GetString(3), reader.GetInt32(0), plan, + DateTimeOffset.Parse(reader.GetString(2))); + } +} + +public sealed class ProvisioningPlanValidationException() + : Exception("Base installation plan does not match the validated job parameters."); diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index f6a81a8..8cb5b30 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -42,9 +42,7 @@ public sealed partial class ControlStore request.ActionType, request.SchemaVersion, request.Parameters.Clone(), - confirmationRequired - ? ProvisioningJobStates.AwaitingConfirmation - : ProvisioningJobStates.Queued, + ProvisioningJobStates.Queued, confirmationRequired, request.AuditReason, actor, @@ -119,12 +117,14 @@ public sealed partial class ControlStore WHERE id = ( SELECT id FROM provisioning_jobs WHERE node_id = $node - AND (state = $queued + AND ((state = $queued + AND (confirmation_required = 0 OR confirmed_at IS NULL)) OR (state = $rolling_back AND current_step = 'rollback-queued')) AND expires_at > $now ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id LIMIT 1) - AND (state = $queued + AND ((state = $queued + AND (confirmation_required = 0 OR confirmed_at IS NULL)) OR (state = $rolling_back AND current_step = 'rollback-queued')) RETURNING *; """; @@ -512,6 +512,7 @@ public sealed partial class ControlStore var updated = current with { State = targetState, + CurrentStep = setConfirmedAt ? "confirmed-queued" : current.CurrentStep, UpdatedAt = now, ConfirmedAt = setConfirmedAt ? now : current.ConfirmedAt, CancelledAt = targetState == ProvisioningJobStates.Cancelled ? now : current.CancelledAt, @@ -521,11 +522,12 @@ public sealed partial class ControlStore update.Transaction = transaction; update.CommandText = """ UPDATE provisioning_jobs SET - state = $state, updated_at = $updated, confirmed_at = $confirmed, + state = $state, current_step = $step, updated_at = $updated, confirmed_at = $confirmed, cancelled_at = $cancelled, version = $version WHERE id = $id AND version = $previous_version; """; AddProvisioningJobParameters(update, updated); + update.Parameters.AddWithValue("$step", updated.CurrentStep); update.Parameters.AddWithValue("$previous_version", current.Version); if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) { diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index a9f5a53..b322519 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -272,6 +272,18 @@ public sealed record SystemBaseInstallPlan( string RebootPolicy, string[] Warnings); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record SystemBaseInstallPlanReportRequest( + SystemBaseInstallPlan Plan, + string IdempotencyKey); + +public sealed record ProvisioningBaseInstallPlanRecord( + string JobId, + string NodeId, + int SchemaVersion, + SystemBaseInstallPlan Plan, + DateTimeOffset CreatedAt); + public static class PreflightDriftStatuses { public const string NotConfigured = "NotConfigured"; @@ -374,6 +386,9 @@ public static class SystemBaseInstallCatalogDefinition public static class SystemBaseInstallSchema { + private static readonly HashSet AllowedPlanWarnings = + new(["apt.missing", "timezone.missing"], StringComparer.Ordinal); + public static bool TryParse(JsonElement json, out SystemBaseInstallParameters? parameters) { try @@ -405,6 +420,27 @@ public static class SystemBaseInstallSchema && parameters.VmSwappiness is >= 0 and <= 200 && parameters.RebootPolicy == "never"; + public static bool IsValidPlan( + SystemBaseInstallParameters parameters, + SystemBaseInstallPlan plan) + => plan is not null + && string.Equals(plan.Timezone, parameters.Timezone, StringComparison.Ordinal) + && string.Equals(plan.Locale, parameters.Locale, StringComparison.Ordinal) + && plan.AptUpdate == parameters.AptUpdate + && plan.AptUpgrade == parameters.AptUpgrade + && plan.Packages is not null + && plan.Packages.SequenceEqual( + SystemBaseInstallCatalogDefinition.ExpandGroups(parameters.PackageGroupIds), + StringComparer.Ordinal) + && string.Equals(plan.SwapMode, parameters.SwapMode, StringComparison.Ordinal) + && plan.SwapSizeMiB == parameters.SwapSizeMiB + && plan.VmSwappiness == parameters.VmSwappiness + && plan.EnableUnattendedUpgrades == parameters.EnableUnattendedUpgrades + && string.Equals(plan.RebootPolicy, parameters.RebootPolicy, StringComparison.Ordinal) + && plan.Warnings is { Length: <= 2 } + && plan.Warnings.Distinct(StringComparer.Ordinal).Count() == plan.Warnings.Length + && plan.Warnings.All(AllowedPlanWarnings.Contains); + private static bool IsSafeTimezone(string? value) => value is { Length: >= 1 and <= 64 } && value[0] is not '/' and not '.' diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index a77cce2..f18a0e2 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -43,6 +43,8 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(SystemPackageGroup[]))] [JsonSerializable(typeof(SystemBaseInstallCatalog))] [JsonSerializable(typeof(SystemBaseInstallPlan))] +[JsonSerializable(typeof(SystemBaseInstallPlanReportRequest))] +[JsonSerializable(typeof(ProvisioningBaseInstallPlanRecord))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index ca4e601..ba7b217 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -132,7 +132,7 @@ public sealed class ControlApiTests : IAsyncDisposable var job = await response.Content.ReadFromJsonAsync( cancellationToken); Assert.NotNull(job); - Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.AwaitingConfirmation, job.State); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, job.State); Assert.Equal(HttpStatusCode.BadRequest, (await client.PostAsJsonAsync( "/api/v1/control/agents/home/provisioning/jobs", new diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 3954266..45eba97 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(7L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(8L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index cfc47bc..d17e00e 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(7L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(8L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); @@ -44,9 +44,9 @@ public sealed class ControlStoreTests : IAsyncDisposable var store = CreateStore(); await store.InitializeAsync(cancellationToken); await EnrollAgentAsync(store, "home", "A1B2", cancellationToken); - using var parameters = JsonDocument.Parse("{}"); + var parameters = CreateBaseInstallParameters(); var request = new ProvisioningJobCreateRequest( - "system.base-install", 1, parameters.RootElement.Clone(), 60, + "system.base-install", 1, parameters, 60, "Prepare test server", Guid.NewGuid().ToString()); var created = await store.CreateProvisioningJobAsync("home", request, "operator", cancellationToken); @@ -54,13 +54,42 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(created.Id, replay.Id); Assert.Equal(created.State, replay.State); - Assert.Equal(ProvisioningJobStates.AwaitingConfirmation, created.State); + Assert.Equal(ProvisioningJobStates.Queued, created.State); Assert.True(created.ConfirmationRequired); await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync( "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken)); var confirmation = new ProvisioningJobCommandRequest( "Approved for test", Guid.NewGuid().ToString()); + await Assert.ThrowsAsync(() => store.ConfirmProvisioningJobAsync( + created.Id, confirmation, "operator", cancellationToken)); + + var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken); + Assert.Equal(ProvisioningJobStates.Preflight, claimed!.State); + var expectedPlan = CreateBaseInstallPlan(); + var invalidPlan = expectedPlan with { Packages = [.. expectedPlan.Packages, "untrusted-package"] }; + await Assert.ThrowsAsync(() => store.RecordBaseInstallPlanAsync( + "home", created.Id, + new SystemBaseInstallPlanReportRequest(invalidPlan, Guid.NewGuid().ToString()), + cancellationToken)); + + var planRequest = new SystemBaseInstallPlanReportRequest( + expectedPlan, Guid.NewGuid().ToString()); + var recorded = await store.RecordBaseInstallPlanAsync( + "home", created.Id, planRequest, cancellationToken); + var planReplay = await store.RecordBaseInstallPlanAsync( + "home", created.Id, planRequest, cancellationToken); + Assert.Equal(recorded!.JobId, planReplay!.JobId); + Assert.Equal(recorded.NodeId, planReplay.NodeId); + Assert.Equal(recorded.CreatedAt, planReplay.CreatedAt); + Assert.Equal(recorded.Plan.Packages, planReplay.Plan.Packages); + Assert.Equal(expectedPlan.Packages, recorded!.Plan.Packages); + Assert.Equal( + ProvisioningJobStates.AwaitingConfirmation, + (await store.GetProvisioningJobAsync(created.Id, cancellationToken))!.State); + var readablePlan = await store.GetBaseInstallPlanAsync(created.Id, cancellationToken); + Assert.Equal(expectedPlan.Packages, readablePlan!.Plan.Packages); + var confirmed = await store.ConfirmProvisioningJobAsync( created.Id, confirmation, "operator", cancellationToken); var confirmationReplay = await store.ConfirmProvisioningJobAsync( @@ -68,7 +97,9 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(confirmed!.Id, confirmationReplay!.Id); Assert.Equal(confirmed.State, confirmationReplay.State); Assert.Equal(ProvisioningJobStates.Queued, confirmed.State); + Assert.Equal("confirmed-queued", confirmed.CurrentStep); Assert.NotNull(confirmed.ConfirmedAt); + Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); var cancelled = await store.CancelProvisioningJobAsync( created.Id, @@ -80,7 +111,7 @@ public sealed class ControlStoreTests : IAsyncDisposable var next = await store.CreateProvisioningJobAsync( "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken); - Assert.Equal(ProvisioningJobStates.AwaitingConfirmation, next.State); + Assert.Equal(ProvisioningJobStates.Queued, next.State); } [Fact] @@ -744,6 +775,19 @@ public sealed class ControlStoreTests : IAsyncDisposable })); } + private static JsonElement CreateBaseInstallParameters() + => JsonSerializer.SerializeToElement( + new SystemBaseInstallParameters( + "UTC", "en_US.UTF-8", true, false, 1, ["core"], + "disabled", null, 60, true, "never"), + SmmJsonContext.Default.SystemBaseInstallParameters); + + private static SystemBaseInstallPlan CreateBaseInstallPlan() + => new( + "UTC", "en_US.UTF-8", true, false, + SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]), + "disabled", null, 60, true, "never", []); + private static async Task EnrollAgentAsync( ControlStore store, string nodeId, -- 2.45.2 From 8ddd4e19a97caaa3e506b268ff341ee8d99ed18d Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Mon, 20 Jul 2026 00:23:30 +0700 Subject: [PATCH 30/31] Test base install plan API flow --- .../ControlApiTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index ba7b217..d56c24c 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -133,6 +133,56 @@ public sealed class ControlApiTests : IAsyncDisposable cancellationToken); Assert.NotNull(job); Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, job.State); + Assert.Equal(HttpStatusCode.Conflict, (await client.PostAsJsonAsync( + $"/api/v1/control/provisioning/jobs/{job.Id}/confirm", + new { reason = "Too early", idempotencyKey = Guid.NewGuid().ToString() }, + cancellationToken)).StatusCode); + + using var agent = _factory.CreateClient(); + agent.DefaultRequestHeaders.Add("X-Test-Identity", "home"); + agent.DefaultRequestHeaders.Add("X-Test-Role", "Agent"); + var claimed = await agent.GetFromJsonAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken); + Assert.Equal(job.Id, claimed!.Id); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State); + var expectedPackages = ServerMonitorManager.Core.SystemBaseInstallCatalogDefinition + .ExpandGroups(["core", "development"]); + Assert.Equal(HttpStatusCode.OK, (await agent.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{job.Id}/base-install-plan", + new + { + plan = new + { + timezone = "UTC", + locale = "en_US.UTF-8", + aptUpdate = true, + aptUpgrade = false, + packages = expectedPackages, + swapMode = "automatic", + swapSizeMiB = (int?)null, + vmSwappiness = 60, + enableUnattendedUpgrades = true, + rebootPolicy = "never", + warnings = Array.Empty() + }, + idempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken)).StatusCode); + var storedPlan = await client.GetFromJsonAsync< + ServerMonitorManager.Core.ProvisioningBaseInstallPlanRecord>( + $"/api/v1/control/provisioning/jobs/{job.Id}/plan", cancellationToken); + Assert.Equal(expectedPackages, storedPlan!.Plan.Packages); + var confirmResponse = await client.PostAsJsonAsync( + $"/api/v1/control/provisioning/jobs/{job.Id}/confirm", + new { reason = "Reviewed generated plan", idempotencyKey = Guid.NewGuid().ToString() }, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, confirmResponse.StatusCode); + var confirmed = await confirmResponse.Content.ReadFromJsonAsync< + ServerMonitorManager.Core.ProvisioningJob>(cancellationToken); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, confirmed!.State); + Assert.Equal("confirmed-queued", confirmed.CurrentStep); + Assert.Equal(HttpStatusCode.NoContent, (await agent.GetAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); Assert.Equal(HttpStatusCode.BadRequest, (await client.PostAsJsonAsync( "/api/v1/control/agents/home/provisioning/jobs", new -- 2.45.2 From 1eae8e1eeaa63e5653b9608aea4c63c56db22355 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Mon, 20 Jul 2026 00:39:07 +0700 Subject: [PATCH 31/31] Authorize confirmed provisioning execution --- README.md | 2 +- docs/provisioning-vpn-requirements.md | 2 + docs/roadmap.md | 1 + .../CertificateAuthority.cs | 44 ++++++ src/ServerMonitorManager.Control/Program.cs | 43 ++++++ .../ProvisioningBaseInstallPlanStore.cs | 71 ++++++++++ src/ServerMonitorManager.Core/Contracts.cs | 17 +++ .../ProvisioningExecutionGrantCodec.cs | 134 ++++++++++++++++++ .../SmmJsonContext.cs | 2 + .../CertificateAuthorityTests.cs | 54 +++++++ .../ControlApiTests.cs | 24 ++++ 11 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs diff --git a/README.md b/README.md index 9907fbd..d339066 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and builds a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Control independently validates and stores that immutable plan before the job enters `AwaitingConfirmation`; confirmed base-install jobs remain safely queued until the mutating executor and factual verification are implemented. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and builds a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Control independently validates and stores that immutable plan before the job enters `AwaitingConfirmation`. After confirmation, Control can issue an idempotent, audited, two-minute ECDSA execution grant bound to the exact job, Node, plan hash, and Control CA; confirmed base-install jobs remain safely queued until the helper consumes this grant and factual verification is implemented. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms. ## License and project policy diff --git a/docs/provisioning-vpn-requirements.md b/docs/provisioning-vpn-requirements.md index 71d15b3..92ebb30 100644 --- a/docs/provisioning-vpn-requirements.md +++ b/docs/provisioning-vpn-requirements.md @@ -289,6 +289,8 @@ GET /api/v1/nodes/{nodeId}/vpn-profiles Agent получает только задания своего Node. Automation identity не создаёт provisioning jobs, не читает VPN secrets и не управляет пользователями. Mutation требует Operator certificate, idempotency key и audit reason. +Подтверждённая опасная операция дополнительно требует короткоживущий execution grant. Control подписывает его ECDSA-ключом Control CA и связывает с `node_id`, `job_id`, action/schema, SHA-256 подтверждённого plan, nonce и сроком действия. Agent получает grant через `POST /api/v1/agents/provisioning/jobs/{jobId}/execution-grant`; root-helper проверяет подпись по закреплённому Control CA до запуска любой мутации. + Каждый action type имеет отдельную versioned JSON Schema. Неизвестные смысловые поля отклоняются на Control, Agent и helper. ## 15. Хранение и аудит diff --git a/docs/roadmap.md b/docs/roadmap.md index c547753..4fe63e5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -118,6 +118,7 @@ - [ ] versioned JSON schemas для остальных action type; - [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`); - [x] двухфазный `system.base-install`: сохранённый проверенный plan до Operator confirmation; +- [x] короткоживущий ECDSA execution grant, привязанный к Node, job и SHA-256 подтверждённого plan; - [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift (`preflight` завершён; остальные action type ещё не подключены); diff --git a/src/ServerMonitorManager.Control/CertificateAuthority.cs b/src/ServerMonitorManager.Control/CertificateAuthority.cs index 6ec9a63..91b0796 100644 --- a/src/ServerMonitorManager.Control/CertificateAuthority.cs +++ b/src/ServerMonitorManager.Control/CertificateAuthority.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using Microsoft.Extensions.Options; +using ServerMonitorManager.Core; namespace ServerMonitorManager.Control; @@ -65,6 +66,49 @@ public sealed class CertificateAuthority : IDisposable notAfter); } + public ProvisioningExecutionGrant SignProvisioningExecutionGrant( + ProvisioningJob job, + SystemBaseInstallPlan plan, + DateTimeOffset issuedAt, + TimeSpan lifetime) + { + if (job.ActionType != "system.base-install" + || job.SchemaVersion != 1 + || job.State != ProvisioningJobStates.Queued + || !job.ConfirmationRequired + || job.ConfirmedAt is null + || lifetime <= TimeSpan.Zero + || lifetime > ProvisioningExecutionGrantCodec.MaximumLifetime) + { + throw new InvalidOperationException( + "Only a confirmed queued base installation job can receive an execution grant."); + } + + var grant = new ProvisioningExecutionGrant( + ProvisioningExecutionGrantCodec.ProtocolVersion, + job.Id, + job.NodeId, + job.ActionType, + job.SchemaVersion, + ProvisioningExecutionGrantCodec.ComputePlanSha256(plan), + issuedAt.ToUnixTimeSeconds(), + issuedAt.Add(lifetime).ToUnixTimeSeconds(), + Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), + ProvisioningExecutionGrantCodec.SignatureAlgorithm, + string.Empty); + using var key = _issuer.GetECDsaPrivateKey(); + if (key is not { KeySize: 256 }) + { + throw new InvalidOperationException( + "Control CA must use an ECDSA P-256 key to sign provisioning execution grants."); + } + var signature = key.SignData( + ProvisioningExecutionGrantCodec.CreateSigningPayload(grant), + HashAlgorithmName.SHA256, + DSASignatureFormat.IeeeP1363FixedFieldConcatenation); + return grant with { Signature = ProvisioningExecutionGrantCodec.EncodeBase64Url(signature) }; + } + public void Dispose() => _issuer.Dispose(); } diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index 0a55761..12a3206 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -498,6 +498,46 @@ agents.MapPost("/provisioning/jobs/{id}/base-install-plan", async ( return Results.BadRequest(new ProblemDetails { Title = exception.Message }); } }); +agents.MapPost("/provisioning/jobs/{id}/execution-grant", async ( + string id, + ProvisioningExecutionGrantRequest request, + HttpContext context, + ControlStore controlStore, + CertificateAuthority authority, + TimeProvider timeProvider, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) + || !NodeIdValidator.IsValid(nodeId) + || !ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["executionGrant"] = ["Invalid job id or idempotency key."] + }); + } + try + { + var grant = await controlStore.IssueBaseInstallExecutionGrantAsync( + nodeId, + id, + request, + (job, plan) => authority.SignProvisioningExecutionGrant( + job, plan, timeProvider.GetUtcNow(), TimeSpan.FromMinutes(2)), + cancellationToken); + return grant is null ? Results.NotFound() : Results.Ok(grant); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => @@ -1056,6 +1096,9 @@ internal static class ProvisioningJobValidator && request.Plan.Warnings is { Length: <= 2 } && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValid(ProvisioningExecutionGrantRequest request) + => IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValidId(string id) => id.Length == 32 && Guid.TryParseExact(id, "N", out _); diff --git a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs index fe93635..8abbe0b 100644 --- a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs @@ -132,6 +132,77 @@ public sealed partial class ControlStore jobId, reader.GetString(3), reader.GetInt32(0), plan, DateTimeOffset.Parse(reader.GetString(2))); } + + public async Task IssueBaseInstallExecutionGrantAsync( + string nodeId, + string jobId, + ProvisioningExecutionGrantRequest request, + Func issueGrant, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-execution-grant:{nodeId}:{jobId}:{request.IdempotencyKey}"; + var requestHash = Fingerprint( + request, SmmJsonContext.Default.ProvisioningExecutionGrantRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken); + if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal)) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (job.ActionType != "system.base-install" + || job.SchemaVersion != 1 + || job.State != ProvisioningJobStates.Queued + || !job.ConfirmationRequired + || job.ConfirmedAt is null + || job.ExpiresAt <= DateTimeOffset.UtcNow) + { + throw new ProvisioningTransitionException(job.State, "ExecutionGrant"); + } + + var planCommand = connection.CreateCommand(); + planCommand.Transaction = transaction; + planCommand.CommandText = """ + SELECT plan_json FROM provisioning_base_install_plans + WHERE job_id = $job AND schema_version = 1; + """; + planCommand.Parameters.AddWithValue("$job", jobId); + var planJson = await planCommand.ExecuteScalarAsync(cancellationToken) as string; + if (planJson is null) + { + throw new ProvisioningTransitionException(job.State, "ExecutionGrant"); + } + var plan = JsonSerializer.Deserialize( + planJson, SmmJsonContext.Default.SystemBaseInstallPlan) + ?? throw new InvalidDataException("Stored base installation plan is invalid."); + var grant = issueGrant(job, plan); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, grant, + SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.execution-grant.issued", jobId, + JsonSerializer.Serialize(new + { + grant.ActionType, + grant.SchemaVersion, + grant.PlanSha256, + grant.ExpiresAtUnixSeconds + }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return grant; + } } public sealed class ProvisioningPlanValidationException() diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index b322519..c0456db 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -284,6 +284,23 @@ public sealed record ProvisioningBaseInstallPlanRecord( SystemBaseInstallPlan Plan, DateTimeOffset CreatedAt); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record ProvisioningExecutionGrant( + string ProtocolVersion, + string JobId, + string NodeId, + string ActionType, + int SchemaVersion, + string PlanSha256, + long IssuedAtUnixSeconds, + long ExpiresAtUnixSeconds, + string Nonce, + string SignatureAlgorithm, + string Signature); + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record ProvisioningExecutionGrantRequest(string IdempotencyKey); + public static class PreflightDriftStatuses { public const string NotConfigured = "NotConfigured"; diff --git a/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs new file mode 100644 index 0000000..e97e9eb --- /dev/null +++ b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs @@ -0,0 +1,134 @@ +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; + +namespace ServerMonitorManager.Core; + +public static class ProvisioningExecutionGrantCodec +{ + public const string ProtocolVersion = "1"; + public const string SignatureAlgorithm = "ECDSA-P256-SHA256-P1363"; + public static readonly TimeSpan MaximumLifetime = TimeSpan.FromMinutes(5); + + public static string ComputePlanSha256(SystemBaseInstallPlan plan) + { + var json = JsonSerializer.SerializeToUtf8Bytes( + plan, SmmJsonContext.Default.SystemBaseInstallPlan); + return Convert.ToHexStringLower(SHA256.HashData(json)); + } + + public static byte[] CreateSigningPayload(ProvisioningExecutionGrant grant) + => Encoding.UTF8.GetBytes(string.Join('\n', + [ + "SMM-PROVISIONING-GRANT-V1", + grant.ProtocolVersion, + grant.JobId, + grant.NodeId, + grant.ActionType, + grant.SchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), + grant.PlanSha256, + grant.IssuedAtUnixSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture), + grant.ExpiresAtUnixSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture), + grant.Nonce, + grant.SignatureAlgorithm + ])); + + public static bool Verify( + ProvisioningExecutionGrant grant, + X509Certificate2 controlAuthority, + string expectedJobId, + string expectedNodeId, + SystemBaseInstallPlan expectedPlan, + DateTimeOffset now) + { + if (grant.ProtocolVersion != ProtocolVersion + || grant.SignatureAlgorithm != SignatureAlgorithm + || grant.JobId != expectedJobId + || grant.NodeId != expectedNodeId + || grant.ActionType != "system.base-install" + || grant.SchemaVersion != 1 + || grant.JobId is not { Length: 32 } + || !grant.JobId.All(Uri.IsHexDigit) + || grant.Nonce is not { Length: 32 } + || !grant.Nonce.All(Uri.IsHexDigit) + || grant.PlanSha256 is not { Length: 64 } + || !grant.PlanSha256.All(Uri.IsHexDigit) + || grant.Signature is not { Length: >= 1 and <= 128 }) + { + return false; + } + + DateTimeOffset issuedAt; + DateTimeOffset expiresAt; + try + { + issuedAt = DateTimeOffset.FromUnixTimeSeconds(grant.IssuedAtUnixSeconds); + expiresAt = DateTimeOffset.FromUnixTimeSeconds(grant.ExpiresAtUnixSeconds); + } + catch (ArgumentOutOfRangeException) + { + return false; + } + if (issuedAt > now.AddSeconds(30) + || expiresAt <= now + || expiresAt <= issuedAt + || expiresAt - issuedAt > MaximumLifetime) + { + return false; + } + + var expectedHash = ComputePlanSha256(expectedPlan); + if (!CryptographicOperations.FixedTimeEquals( + Encoding.ASCII.GetBytes(expectedHash), + Encoding.ASCII.GetBytes(grant.PlanSha256.ToLowerInvariant()))) + { + return false; + } + + byte[] signature; + try + { + signature = DecodeBase64Url(grant.Signature); + } + catch (FormatException) + { + return false; + } + if (signature.Length != 64) + { + return false; + } + + using var key = controlAuthority.GetECDsaPublicKey(); + return key is { KeySize: 256 } + && key.VerifyData( + CreateSigningPayload(grant), signature, HashAlgorithmName.SHA256, + DSASignatureFormat.IeeeP1363FixedFieldConcatenation); + } + + public static string EncodeBase64Url(ReadOnlySpan value) + => Convert.ToBase64String(value) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + + private static byte[] DecodeBase64Url(string value) + { + if (value.Length is < 1 or > 128 + || value.Any(character => !char.IsAsciiLetterOrDigit(character) + && character is not '-' and not '_')) + { + throw new FormatException("Invalid base64url value."); + } + var padded = value.Replace('-', '+').Replace('_', '/'); + padded += (padded.Length % 4) switch + { + 2 => "==", + 3 => "=", + 0 => string.Empty, + _ => throw new FormatException("Invalid base64url value.") + }; + return Convert.FromBase64String(padded); + } +} diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index f18a0e2..88e6a38 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -45,6 +45,8 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(SystemBaseInstallPlan))] [JsonSerializable(typeof(SystemBaseInstallPlanReportRequest))] [JsonSerializable(typeof(ProvisioningBaseInstallPlanRecord))] +[JsonSerializable(typeof(ProvisioningExecutionGrant))] +[JsonSerializable(typeof(ProvisioningExecutionGrantRequest))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs index ceda103..210f2ac 100644 --- a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs @@ -2,6 +2,7 @@ using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using Microsoft.Extensions.Options; using ServerMonitorManager.Control; +using ServerMonitorManager.Core; using Xunit; namespace ServerMonitorManager.Control.Tests; @@ -49,6 +50,59 @@ public sealed class CertificateAuthorityTests : IDisposable oid => oid.Value == "1.3.6.1.5.5.7.3.2"); } + [Fact] + public void ProvisioningExecutionGrantIsBoundToConfirmedJobPlanAndExpiry() + { + Directory.CreateDirectory(_directory); + var caPath = Path.Combine(_directory, "grant-control-ca.pfx"); + using var caKey = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var caRequest = new CertificateRequest("CN=SMM Grant Test CA", caKey, HashAlgorithmName.SHA256); + caRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true)); + caRequest.CertificateExtensions.Add(new X509KeyUsageExtension( + X509KeyUsageFlags.KeyCertSign | X509KeyUsageFlags.CrlSign | X509KeyUsageFlags.DigitalSignature, + true)); + using var ca = caRequest.CreateSelfSigned( + DateTimeOffset.UtcNow.AddMinutes(-1), + DateTimeOffset.UtcNow.AddYears(2)); + File.WriteAllBytes(caPath, ca.Export(X509ContentType.Pfx)); + using var authority = new CertificateAuthority(Options.Create(new ControlOptions + { + DatabasePath = Path.Combine(_directory, "unused-grant.db"), + CertificateAuthorityPath = caPath + })); + + var now = DateTimeOffset.UtcNow; + var parameters = System.Text.Json.JsonSerializer.SerializeToElement( + new SystemBaseInstallParameters( + "UTC", "en_US.UTF-8", true, false, 1, ["core"], + "disabled", null, 60, true, "never"), + SmmJsonContext.Default.SystemBaseInstallParameters); + var job = new ProvisioningJob( + Guid.NewGuid().ToString("N"), "home", "system.base-install", 1, parameters, + ProvisioningJobStates.Queued, true, "test", "operator", + now.AddMinutes(-1), now, now.AddMinutes(30), now, null, 4, 25, + "confirmed-queued", null); + var plan = new SystemBaseInstallPlan( + "UTC", "en_US.UTF-8", true, false, + SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]), + "disabled", null, 60, true, "never", []); + + var grant = authority.SignProvisioningExecutionGrant( + job, plan, now, TimeSpan.FromMinutes(2)); + + Assert.True(ProvisioningExecutionGrantCodec.Verify( + grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now)); + Assert.False(ProvisioningExecutionGrantCodec.Verify( + grant, authority.PublicCertificate, job.Id, "other", plan, now)); + Assert.False(ProvisioningExecutionGrantCodec.Verify( + grant, authority.PublicCertificate, job.Id, job.NodeId, + plan with { Packages = [.. plan.Packages, "untrusted-package"] }, now)); + Assert.False(ProvisioningExecutionGrantCodec.Verify( + grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now.AddMinutes(3))); + Assert.Throws(() => authority.SignProvisioningExecutionGrant( + job with { ConfirmedAt = null }, plan, now, TimeSpan.FromMinutes(2))); + } + public void Dispose() { if (Directory.Exists(_directory)) diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index d56c24c..0c9531a 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -181,6 +181,30 @@ public sealed class ControlApiTests : IAsyncDisposable ServerMonitorManager.Core.ProvisioningJob>(cancellationToken); Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, confirmed!.State); Assert.Equal("confirmed-queued", confirmed.CurrentStep); + var grantRequest = new { idempotencyKey = Guid.NewGuid().ToString() }; + var grantResponse = await agent.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant", + grantRequest, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, grantResponse.StatusCode); + var grant = await grantResponse.Content.ReadFromJsonAsync< + ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken); + var grantReplayResponse = await agent.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant", + grantRequest, + cancellationToken); + var grantReplay = await grantReplayResponse.Content.ReadFromJsonAsync< + ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken); + Assert.Equal(grant!.Signature, grantReplay!.Signature); + var authority = _factory.Services.GetRequiredService< + ServerMonitorManager.Control.CertificateAuthority>(); + Assert.True(ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.Verify( + grant, + authority.PublicCertificate, + job.Id, + "home", + storedPlan.Plan, + DateTimeOffset.UtcNow)); Assert.Equal(HttpStatusCode.NoContent, (await agent.GetAsync( "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); Assert.Equal(HttpStatusCode.BadRequest, (await client.PostAsJsonAsync( -- 2.45.2