diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml
index f9b52ab..e442118 100644
--- a/.github/workflows/linux-control-agent.yml
+++ b/.github/workflows/linux-control-agent.yml
@@ -10,7 +10,7 @@ permissions:
jobs:
build-and-test:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -37,8 +37,102 @@ jobs:
bash -n tests/acceptance/three-server-mesh.sh
shellcheck --severity=error tests/acceptance/three-server-mesh.sh
+ - name: Verify standalone bootstrap
+ run: |
+ bash -n deploy/ochenstarik-server-monitor-manager.sh
+ bash -n deploy/ochenstarik-smm-policy-apply
+ bash -n deploy/ochenstarik-smm-emergency
+ bash -n tests/bootstrap/run-native-systemd-smoke.sh
+ bash -n tests/bootstrap/run-systemd-container-smoke.sh
+ shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh
+ shellcheck --severity=error deploy/ochenstarik-smm-policy-apply
+ shellcheck --severity=error deploy/ochenstarik-smm-emergency
+ shellcheck --severity=error tests/bootstrap/run-native-systemd-smoke.sh
+ shellcheck --severity=error tests/bootstrap/run-systemd-container-smoke.sh
+ bash tests/bootstrap/test-bootstrap-contract.sh
+
- name: Publish agent amd64
run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-x64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
- name: Publish agent arm64
run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
+
+ - name: Publish provisioning helper amd64
+ run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime linux-x64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
+
+ - name: Publish provisioning helper arm64
+ run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
+
+ - name: Build systemd smoke release
+ run: |
+ dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj \
+ --configuration Release --runtime linux-x64 --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/agent
+ dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \
+ --configuration Release --runtime linux-x64 --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/control
+ dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj \
+ --configuration Release --runtime linux-x64 --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o smoke/provisioning-helper
+ install -d smoke/deploy smoke/bootstrap
+ install -m 0644 deploy/ochenstarik-smm-control.service smoke/deploy/
+ install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/
+ install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service smoke/deploy/
+ install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/
+ install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/
+ install -m 0755 deploy/ochenstarik-smm-emergency smoke/deploy/
+ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/
+ tar -C smoke -czf smoke-release.tar.gz agent control provisioning-helper deploy bootstrap
+ sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256
+
+ - name: Verify repeated systemd installation
+ run: |
+ set -Eeuo pipefail
+ smoke_log=/tmp/smm-systemd-smoke.log
+ report_failure() {
+ local title="$1" message main_pid
+ sudo systemctl status --no-pager ochenstarik-smm-control.service >>"$smoke_log" 2>&1 || true
+ sudo journalctl --no-pager -u ochenstarik-smm-control.service -n 40 >>"$smoke_log" 2>&1 || true
+ sudo ss -ltnp >>"$smoke_log" 2>&1 || true
+ main_pid="$(sudo systemctl show -p MainPID --value ochenstarik-smm-control.service 2>/dev/null || true)"
+ if [[ "$main_pid" =~ ^[1-9][0-9]*$ ]]; then
+ sudo sh -c "tr '\0' '\n' >"$smoke_log" 2>&1 || true
+ sudo sh -c "printf 'wchan='; cat /proc/$main_pid/wchan" >>"$smoke_log" 2>&1 || true
+ fi
+ message="$(tail -c 6000 "$smoke_log" | sed ':a;N;$!ba;s/%/%25/g;s/\r/%0D/g;s/\n/%0A/g')"
+ printf '::error title=%s::%s\n' "$title" "$message"
+ exit 1
+ }
+ sudo deploy/ochenstarik-server-monitor-manager.sh install-control \
+ smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \
+ || report_failure 'Initial Control installation failed'
+ sudo test -x /usr/local/sbin/ochenstarik-smm-emergency \
+ || report_failure 'Emergency recovery command was not installed'
+ sudo /usr/local/sbin/ochenstarik-smm-emergency status >>"$smoke_log" 2>&1 \
+ || report_failure 'Emergency recovery status failed'
+ for _ in {1..30}; do
+ sudo curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ https://127.0.0.1:7443/healthz && break
+ sleep 1
+ done
+ sudo curl --fail --silent --show-error \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \
+ || report_failure 'Initial Control healthcheck failed'
+ sudo deploy/ochenstarik-server-monitor-manager.sh install-control \
+ smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \
+ || report_failure 'Repeated Control installation failed'
+ sudo systemctl restart ochenstarik-smm-control.service \
+ || report_failure 'Control restart failed'
+ sudo systemctl is-active --quiet ochenstarik-smm-control.service \
+ || report_failure 'Control is inactive after restart'
+ sudo curl --fail --silent --show-error \
+ --retry 10 --retry-all-errors --retry-delay 1 \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ https://127.0.0.1:7443/healthz >>"$smoke_log" 2>&1 \
+ || report_failure 'Repeated Control healthcheck failed'
+
+ - name: Clean up systemd smoke installation
+ if: always()
+ run: sudo deploy/ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control || true
diff --git a/.github/workflows/linux-platform-matrix.yml b/.github/workflows/linux-platform-matrix.yml
new file mode 100644
index 0000000..b018276
--- /dev/null
+++ b/.github/workflows/linux-platform-matrix.yml
@@ -0,0 +1,153 @@
+name: Linux platform matrix
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - '.github/workflows/linux-platform-matrix.yml'
+ - 'deploy/**'
+ - 'src/ServerMonitorManager.Agent/**'
+ - 'src/ServerMonitorManager.Control/**'
+ - 'src/ServerMonitorManager.Core/**'
+ - 'tests/bootstrap/**'
+ push:
+ branches: [main]
+ paths:
+ - '.github/workflows/linux-platform-matrix.yml'
+ - 'deploy/**'
+ - 'src/ServerMonitorManager.Agent/**'
+ - 'src/ServerMonitorManager.Control/**'
+ - 'src/ServerMonitorManager.Core/**'
+ - 'tests/bootstrap/**'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: linux-platform-matrix-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ release-archive:
+ name: Release archive (${{ matrix.runtime }})
+ runs-on: ubuntu-24.04
+ strategy:
+ matrix:
+ runtime: [linux-x64, linux-arm64]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+
+ - name: Set up .NET 10
+ uses: actions/setup-dotnet@v5
+ with:
+ dotnet-version: 10.0.x
+
+ - name: Publish compatible release
+ run: |
+ set -Eeuo pipefail
+ dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj \
+ --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/agent
+ dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj \
+ --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control
+ dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj \
+ --configuration Release --runtime '${{ matrix.runtime }}' --self-contained true \
+ -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/provisioning-helper
+ install -d out/deploy out/bootstrap
+ install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/
+ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/
+ install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/
+ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/
+ archive='server-monitor-manager-${{ matrix.runtime }}.tar.gz'
+ tar -C out -czf "$archive" agent control provisioning-helper deploy bootstrap
+ sha256sum "$archive" >"$archive.sha256"
+
+ - name: Upload release archive
+ uses: actions/upload-artifact@v6
+ with:
+ name: server-monitor-manager-${{ matrix.runtime }}-matrix
+ path: |
+ server-monitor-manager-${{ matrix.runtime }}.tar.gz
+ server-monitor-manager-${{ matrix.runtime }}.tar.gz.sha256
+
+ ubuntu-vm:
+ name: ${{ matrix.name }} native VM
+ needs: release-archive
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Ubuntu 22.04 x64
+ runner: ubuntu-22.04
+ runtime: linux-x64
+ - name: Ubuntu 24.04 x64
+ runner: ubuntu-24.04
+ runtime: linux-x64
+ - name: Ubuntu 22.04 arm64
+ runner: ubuntu-22.04-arm
+ runtime: linux-arm64
+ - name: Ubuntu 24.04 arm64
+ runner: ubuntu-24.04-arm
+ runtime: linux-arm64
+ runs-on: ${{ matrix.runner }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+
+ - name: Download release archive
+ uses: actions/download-artifact@v8
+ with:
+ name: server-monitor-manager-${{ matrix.runtime }}-matrix
+ path: artifacts
+
+ - name: Install twice and restart services
+ run: |
+ bash tests/bootstrap/run-native-systemd-smoke.sh \
+ 'artifacts/server-monitor-manager-${{ matrix.runtime }}.tar.gz' \
+ deploy/ochenstarik-server-monitor-manager.sh
+
+ debian-systemd:
+ name: ${{ matrix.name }} systemd restart
+ needs: release-archive
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Debian 12 x64
+ runner: ubuntu-24.04
+ runtime: linux-x64
+ image: debian:12
+ - name: Debian 13 x64
+ runner: ubuntu-24.04
+ runtime: linux-x64
+ image: debian:13
+ - name: Debian 12 arm64
+ runner: ubuntu-24.04-arm
+ runtime: linux-arm64
+ image: debian:12
+ - name: Debian 13 arm64
+ runner: ubuntu-24.04-arm
+ runtime: linux-arm64
+ image: debian:13
+ runs-on: ${{ matrix.runner }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+
+ - name: Download release archive
+ uses: actions/download-artifact@v8
+ with:
+ name: server-monitor-manager-${{ matrix.runtime }}-matrix
+ path: artifacts
+
+ - name: Install twice and restart systemd container
+ run: |
+ bash tests/bootstrap/run-systemd-container-smoke.sh \
+ '${{ matrix.image }}' \
+ 'artifacts/server-monitor-manager-${{ matrix.runtime }}.tar.gz' \
+ deploy/ochenstarik-server-monitor-manager.sh
diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml
index 87475c1..e8b263a 100644
--- a/.github/workflows/linux-release.yml
+++ b/.github/workflows/linux-release.yml
@@ -10,6 +10,60 @@ permissions:
contents: write
jobs:
+ bootstrap:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+
+ - name: Validate bootstrap
+ run: |
+ bash -n deploy/ochenstarik-server-monitor-manager.sh
+ bash -n deploy/ochenstarik-smm-policy-apply
+ bash -n deploy/ochenstarik-smm-emergency
+ bash -n tests/bootstrap/run-native-systemd-smoke.sh
+ bash -n tests/bootstrap/run-systemd-container-smoke.sh
+ shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh
+ shellcheck --severity=error deploy/ochenstarik-smm-policy-apply
+ shellcheck --severity=error deploy/ochenstarik-smm-emergency
+ shellcheck --severity=error tests/bootstrap/run-native-systemd-smoke.sh
+ shellcheck --severity=error tests/bootstrap/run-systemd-container-smoke.sh
+ bash tests/bootstrap/test-bootstrap-contract.sh
+
+ - name: Package bootstrap
+ shell: bash
+ run: |
+ set -Eeuo pipefail
+ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh ochenstarik-server-monitor-manager.sh
+ sha256sum ochenstarik-server-monitor-manager.sh > ochenstarik-server-monitor-manager.sh.sha256
+ bootstrap_sha="$(sha256sum ochenstarik-server-monitor-manager.sh | awk '{print $1}')"
+ jq -n \
+ --arg schema "smm-bootstrap-manifest/v1" \
+ --arg version "${GITHUB_REF_NAME}" \
+ --arg bootstrap "ochenstarik-server-monitor-manager.sh" \
+ --arg bootstrap_sha256 "$bootstrap_sha" \
+ '{schema: $schema, version: $version, bootstrap: $bootstrap, bootstrap_sha256: $bootstrap_sha256, supported_runtimes: ["linux-x64", "linux-arm64"]}' \
+ > server-monitor-manager-bootstrap-manifest.json
+
+ - name: Upload bootstrap artifact
+ uses: actions/upload-artifact@v6
+ with:
+ name: server-monitor-manager-bootstrap
+ path: |
+ ochenstarik-server-monitor-manager.sh
+ ochenstarik-server-monitor-manager.sh.sha256
+ server-monitor-manager-bootstrap-manifest.json
+
+ - name: Attach bootstrap to GitHub Release
+ if: startsWith(github.ref, 'refs/tags/')
+ uses: softprops/action-gh-release@v2
+ with:
+ prerelease: ${{ contains(github.ref_name, '-') }}
+ files: |
+ ochenstarik-server-monitor-manager.sh
+ ochenstarik-server-monitor-manager.sh.sha256
+ server-monitor-manager-bootstrap-manifest.json
+
publish:
runs-on: ubuntu-latest
strategy:
@@ -30,12 +84,23 @@ jobs:
- name: Publish control
run: dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control
+ - name: Publish provisioning helper
+ run: dotnet publish src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/provisioning-helper
+
- name: Package
shell: bash
run: |
set -Eeuo pipefail
+ install -d out/deploy out/bootstrap
+ install -m 0644 deploy/ochenstarik-smm-control.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-provisioning-helper.service out/deploy/
+ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/
+ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/
+ install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/
+ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/
archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz"
- tar -C out -czf "$archive" agent control
+ tar -C out -czf "$archive" agent control provisioning-helper deploy bootstrap
sha256sum "$archive" > "$archive.sha256"
- name: Upload artifact
diff --git a/README.md b/README.md
index 57b626f..d339066 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ The control plane and data plane are separated:
See [architecture](docs/architecture.md), [security model](docs/security-model.md), [roadmap](docs/roadmap.md), [Linux bootstrap contract](docs/installer-contract.md), and the [Provisioning and Xray specification](docs/provisioning-vpn-requirements.md).
-Operational procedures are documented in [Control backup and recovery](docs/control-backup.md) and the [three-server acceptance test](docs/three-server-acceptance.md).
+Operational procedures are documented in [Linux bootstrap](docs/linux-bootstrap.md), [Control backup and recovery](docs/control-backup.md), and the [three-server acceptance test](docs/three-server-acceptance.md).
## Repository layout
@@ -100,7 +100,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile
The current development branch implements dedicated Windows pages for Servers, Links, Sessions, and Settings; SSH monitoring; directional Links; one-time enrollment; separate mTLS Agent, Operator, and source-scoped Automation identities; certificate revocation/re-enrollment; SQLite control state; audit; authenticated event streaming; Windows Control API integration; and a bounded durable Agent buffer with downsampling.
-Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. CI exercises the Control-to-helper process boundary, HTTP authorization, Agent parsing, Desktop contracts, and a 100-Node concurrent heartbeat/replay scenario. Still required are the project-owned bootstrap, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Provisioning/Xray, and clients for additional platforms.
+Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Control also expires TTL Links through the firewall helper, prunes bounded operational data, versions its SQLite schema, and creates verified backups of SQLite state and the Control CA. The Provisioning control plane persists versioned jobs, enforces TTL/audit/idempotency and one active job per Node, and supports confirmation, progress, reconciliation, retry, and rollback states. A restricted root helper accepts only versioned, module-hashed allowlisted requests through a local Unix socket. It executes read-only Linux `preflight` and builds a deterministic, non-mutating `system.base-install` plan from catalog-backed parameters. Control independently validates and stores that immutable plan before the job enters `AwaitingConfirmation`. After confirmation, Control can issue an idempotent, audited, two-minute ECDSA execution grant bound to the exact job, Node, plan hash, and Control CA; confirmed base-install jobs remain safely queued until the helper consumes this grant and factual verification is implemented. Typed desired and factual states are persisted, versioned, idempotent, and compared through fixed drift codes exposed to Operators. CI exercises process boundaries, authorization, Agent parsing, Desktop contracts, and concurrent heartbeat/replay. Still required are mutating Provisioning actions with factual-state verification, physical WireGuard/nftables/reboot acceptance, trusted public code signing, Xray, and clients for additional platforms.
## License and project policy
diff --git a/ServerMonitorManager.slnx b/ServerMonitorManager.slnx
index 25c13cb..c25a2d1 100644
--- a/ServerMonitorManager.slnx
+++ b/ServerMonitorManager.slnx
@@ -2,5 +2,6 @@
+
diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh
new file mode 100755
index 0000000..34825df
--- /dev/null
+++ b/deploy/ochenstarik-server-monitor-manager.sh
@@ -0,0 +1,911 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+readonly PROGRAM="ochenstarik-server-monitor-manager"
+readonly PROGRAM_VERSION="0.2.0-dev"
+readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager"
+readonly LIB_DIR="/usr/local/lib/ochenstarik-server-monitor-manager"
+readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager"
+readonly BACKUP_DIR="${STATE_DIR}/bootstrap-backups"
+readonly CONTROL_USER="ochenstarik-smm-control"
+readonly AGENT_USER="ochenstarik-smm-agent"
+readonly CONTROL_UNIT="ochenstarik-smm-control.service"
+readonly AGENT_UNIT="ochenstarik-smm-agent.service"
+readonly PROVISIONING_HELPER_UNIT="ochenstarik-smm-provisioning-helper.service"
+readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply"
+readonly EMERGENCY_COMMAND="/usr/local/sbin/ochenstarik-smm-emergency"
+readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control"
+readonly MESH_DIR="${STATE_DIR}/mesh"
+readonly WG_DIR="${ETC_DIR}/wireguard"
+readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service"
+readonly MESH_NETWORK="10.77.0.0/24"
+readonly HUB_MESH_ADDRESS="10.77.0.1/24"
+
+TEMP_DIR=""
+MESH_PEER_CODE=""
+
+log() { printf '%s\n' "[$PROGRAM] $*"; }
+fail() { printf '%s\n' "[$PROGRAM] ERROR: $*" >&2; exit 1; }
+
+cleanup() {
+ if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then
+ rm -rf -- "$TEMP_DIR"
+ fi
+}
+trap cleanup EXIT
+
+usage() {
+ cat <<'EOF'
+Server Monitor Manager Linux bootstrap
+
+Usage:
+ ochenstarik-server-monitor-manager.sh preflight
+ ochenstarik-server-monitor-manager.sh verify-release ARCHIVE
+ ochenstarik-server-monitor-manager.sh install-control ARCHIVE PUBLIC_HOST [HTTPS_PORT]
+ ochenstarik-server-monitor-manager.sh install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT
+ ochenstarik-server-monitor-manager.sh install-node ARCHIVE
+ ochenstarik-server-monitor-manager.sh mesh-init PUBLIC_ENDPOINT [WG_PORT]
+ ochenstarik-server-monitor-manager.sh peer-add SMMPEER1_CODE
+ ochenstarik-server-monitor-manager.sh mesh-status
+ ochenstarik-server-monitor-manager.sh update-control ARCHIVE
+ ochenstarik-server-monitor-manager.sh update-agent ARCHIVE
+ ochenstarik-server-monitor-manager.sh rollback control|agent [BACKUP_ID]
+ ochenstarik-server-monitor-manager.sh node-code NODE_ID
+ ochenstarik-server-monitor-manager.sh node-token NODE_ID
+ ochenstarik-server-monitor-manager.sh control-ca-fingerprint
+ ochenstarik-server-monitor-manager.sh status
+ ochenstarik-server-monitor-manager.sh uninstall-agent [--purge]
+ ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control
+ ochenstarik-server-monitor-manager.sh version
+
+ARCHIVE must have a matching ARCHIVE.sha256 file. Agent enrollment reads the
+one-time token from SMM_ENROLL_TOKEN or from a hidden local prompt; it is never
+written to agent.env.
+EOF
+}
+
+base64url_encode() {
+ base64 -w 0 | tr '+/' '-_' | tr -d '='
+}
+
+base64url_decode() {
+ local value="$1" remainder
+ [[ "$value" =~ ^[A-Za-z0-9_-]+$ ]] || fail "Enrollment code contains invalid base64url data."
+ remainder=$(( ${#value} % 4 ))
+ case "$remainder" in
+ 0) ;;
+ 2) value+="==" ;;
+ 3) value+="=" ;;
+ *) fail "Enrollment code contains invalid base64url length." ;;
+ esac
+ printf '%s' "$value" | tr '_-' '/+' | base64 -d
+}
+
+require_root() {
+ [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "This action must run as root (use sudo)."
+}
+
+require_command() {
+ command -v "$1" >/dev/null 2>&1 || fail "Required command is missing: $1"
+}
+
+validate_platform() {
+ [[ -r /etc/os-release ]] || fail "/etc/os-release is missing."
+ # shellcheck disable=SC1091
+ . /etc/os-release
+ case "${ID:-}" in
+ ubuntu)
+ case "${VERSION_ID:-}" in 22.04|24.04) ;; *) fail "Unsupported Ubuntu version: ${VERSION_ID:-unknown}" ;; esac
+ ;;
+ debian)
+ case "${VERSION_ID:-}" in 12|13) ;; *) fail "Unsupported Debian version: ${VERSION_ID:-unknown}" ;; esac
+ ;;
+ *) fail "Unsupported distribution: ${ID:-unknown}" ;;
+ esac
+ case "$(uname -m)" in
+ x86_64|aarch64|arm64) ;;
+ *) fail "Unsupported architecture: $(uname -m)" ;;
+ esac
+ [[ "$(ps -p 1 -o comm=)" == "systemd" ]] || fail "systemd must be PID 1."
+}
+
+validate_node_id() {
+ [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]] \
+ || fail "Node id must contain 1-63 lowercase letters, digits, or hyphens."
+}
+
+validate_port() {
+ [[ "$1" =~ ^[0-9]+$ ]] && (( 10#$1 >= 1 && 10#$1 <= 65535 )) \
+ || fail "Port must be in range 1-65535."
+}
+
+validate_control_url() {
+ [[ "$1" =~ ^https://[A-Za-z0-9._:\[\]-]+(:[0-9]{1,5})?/?$ ]] \
+ || fail "Control URL must be an https URL without a path or credentials."
+}
+
+verify_archive() {
+ local archive="$1" checksum_file expected actual entry
+ [[ -f "$archive" ]] || fail "Archive not found: $archive"
+ checksum_file="${archive}.sha256"
+ [[ -f "$checksum_file" ]] || fail "Checksum file not found: $checksum_file"
+ expected="$(awk 'NR == 1 { print $1 }' "$checksum_file")"
+ [[ "$expected" =~ ^[0-9a-fA-F]{64}$ ]] || fail "Invalid checksum file: $checksum_file"
+ actual="$(sha256sum "$archive" | awk '{ print $1 }')"
+ [[ "${actual,,}" == "${expected,,}" ]] || fail "Archive checksum mismatch."
+
+ while IFS= read -r entry; do
+ [[ -n "$entry" ]] || continue
+ [[ "$entry" != /* && "$entry" != *".."* ]] || fail "Unsafe archive entry: $entry"
+ case "$entry" in
+ agent|agent/*|control|control/*|provisioning-helper|provisioning-helper/*|deploy|deploy/*|bootstrap|bootstrap/*) ;;
+ *) fail "Unexpected archive entry: $entry" ;;
+ esac
+ done < <(tar -tzf "$archive")
+}
+
+extract_archive() {
+ local archive="$1"
+ verify_archive "$archive"
+ TEMP_DIR="$(mktemp -d -t smm-bootstrap.XXXXXXXX)"
+ chmod 700 "$TEMP_DIR"
+ tar -xzf "$archive" -C "$TEMP_DIR" --no-same-owner --no-same-permissions
+ [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive."
+ [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive."
+ [[ -f "$TEMP_DIR/deploy/$PROVISIONING_HELPER_UNIT" ]] || fail "Provisioning helper systemd unit is missing from archive."
+ [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive."
+ [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing from archive."
+ [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency command is missing from archive."
+}
+
+verify_release_payload() {
+ local archive="$1"
+ require_command sha256sum
+ require_command tar
+ extract_archive "$archive"
+ [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing."
+ [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing."
+ [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] || fail "Provisioning helper binary is missing."
+ [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing."
+ [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency recovery command is missing."
+ [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing."
+ [[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing."
+ log "Release archive and checksum are valid."
+}
+
+ensure_system_user() {
+ local user="$1"
+ if ! getent group "$user" >/dev/null; then
+ groupadd --system "$user"
+ fi
+ if ! id "$user" >/dev/null 2>&1; then
+ useradd --system --gid "$user" --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin "$user"
+ fi
+}
+
+ensure_mesh_packages() {
+ local missing=0 command_name
+ for command_name in wg wg-quick nft ip; do
+ command -v "$command_name" >/dev/null 2>&1 || missing=1
+ done
+ (( missing == 0 )) && return
+ require_command apt-get
+ log "Installing WireGuard/nftables dependencies."
+ DEBIAN_FRONTEND=noninteractive apt-get update
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ wireguard-tools nftables iproute2
+}
+
+install_tree_atomic() {
+ local source="$1" destination="$2" owner="$3" staging
+ [[ -d "$source" ]] || fail "Release payload is missing: $source"
+ staging="${destination}.new.$$"
+ rm -rf -- "$staging"
+ install -d -m 0755 "$staging"
+ cp -a -- "$source/." "$staging/"
+ chown -R "$owner" "$staging"
+ find "$staging" -type d -exec chmod 0755 {} +
+ find "$staging" -type f -exec chmod 0644 {} +
+ find "$staging" -type f -name 'ochenstarik-smm-*' -exec chmod 0755 {} +
+ rm -rf -- "$destination"
+ mv -- "$staging" "$destination"
+}
+
+create_backup() {
+ local role="$1" backup_id archive list=()
+ backup_id="$(date -u +%Y%m%dT%H%M%SZ)-${role}-$RANDOM"
+ install -d -m 0700 "$BACKUP_DIR"
+ archive="$BACKUP_DIR/${backup_id}.tar.gz"
+ case "$role" in
+ control)
+ list=(
+ "usr/local/lib/ochenstarik-server-monitor-manager/control"
+ "etc/ochenstarik-server-monitor-manager/control.env"
+ "etc/ochenstarik-server-monitor-manager/control-ca.pfx"
+ "etc/ochenstarik-server-monitor-manager/control-server.pfx"
+ "etc/systemd/system/$CONTROL_UNIT"
+ "usr/local/libexec/ochenstarik-smm-policy-apply"
+ "etc/sudoers.d/ochenstarik-smm-control"
+ )
+ ;;
+ agent)
+ list=(
+ "usr/local/lib/ochenstarik-server-monitor-manager/agent"
+ "usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper"
+ "etc/ochenstarik-server-monitor-manager/agent.env"
+ "etc/ochenstarik-server-monitor-manager/control-ca.crt"
+ "etc/systemd/system/$AGENT_UNIT"
+ "etc/systemd/system/$PROVISIONING_HELPER_UNIT"
+ )
+ ;;
+ *) fail "Unknown backup role: $role" ;;
+ esac
+ local existing=() item
+ for item in "${list[@]}"; do
+ [[ -e "/$item" ]] && existing+=("$item")
+ done
+ if (( ${#existing[@]} == 0 )); then
+ printf '%s\n' "empty" >"$BACKUP_DIR/${backup_id}.empty"
+ else
+ tar -C / -czf "$archive" -- "${existing[@]}"
+ chmod 0600 "$archive"
+ fi
+ printf '%s\n' "$backup_id"
+}
+
+install_unit() {
+ local source="$1" unit="$2"
+ install -m 0644 "$source" "/etc/systemd/system/$unit"
+ systemctl daemon-reload
+}
+
+write_mesh_firewall() {
+ cat >"$ETC_DIR/mesh.nft" <<'EOF'
+table inet ochenstarik_smm {
+ chain links {
+ ct state established,related accept
+ counter drop
+ }
+
+ chain mesh_forward {
+ type filter hook forward priority filter; policy accept;
+ iifname "smm0" oifname "smm0" jump links
+ }
+ }
+EOF
+ chmod 0644 "$ETC_DIR/mesh.nft"
+ if ! nft list table inet ochenstarik_smm >/dev/null 2>&1; then
+ nft --check -f "$ETC_DIR/mesh.nft"
+ fi
+}
+
+read_mesh_value() {
+ local key="$1"
+ [[ -r "$ETC_DIR/mesh.env" ]] || fail "Mesh Hub is not initialized."
+ awk -F '=' -v key="$key" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "$ETC_DIR/mesh.env"
+}
+
+render_hub_wireguard_config() {
+ local private_key endpoint port node_id address public_key status
+ private_key="$(cat "$WG_DIR/hub.key")"
+ endpoint="$(read_mesh_value HUB_ENDPOINT)"
+ port="${endpoint##*:}"
+ cat >"/etc/wireguard/smm0.conf" <>"/etc/wireguard/smm0.conf" <"$WG_DIR/hub.key"
+ fi
+ hub_private="$(cat "$WG_DIR/hub.key")"
+ hub_public="$(printf '%s' "$hub_private" | wg pubkey)"
+ printf '%s\n' "$hub_public" >"$WG_DIR/hub.pub"
+ chmod 0600 "$WG_DIR/hub.key"
+ chmod 0644 "$WG_DIR/hub.pub"
+ cat >"$ETC_DIR/mesh.env" <"/etc/sysctl.d/90-ochenstarik-smm-mesh.conf"
+ sysctl --system >/dev/null
+ write_mesh_firewall
+ [[ -f "$LIB_DIR/control/ochenstarik-smm-control" ]] \
+ || log "Warning: Control is not installed yet; mesh peer codes require Control enrollment."
+ if [[ -f "${TEMP_DIR:-}/deploy/$FIREWALL_UNIT" ]]; then
+ install_unit "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$FIREWALL_UNIT"
+ elif [[ -f "$LIB_DIR/bootstrap/$FIREWALL_UNIT" ]]; then
+ install_unit "$LIB_DIR/bootstrap/$FIREWALL_UNIT" "$FIREWALL_UNIT"
+ else
+ fail "Mesh firewall systemd unit is unavailable; reinstall Control from the current release."
+ fi
+ systemctl enable "$FIREWALL_UNIT"
+ systemctl restart "$FIREWALL_UNIT"
+ render_hub_wireguard_config
+ systemctl enable wg-quick@smm0.service
+ systemctl restart wg-quick@smm0.service
+ log "Mesh Hub initialized at $public_endpoint:$port with $MESH_NETWORK."
+ log "WireGuard public key: $hub_public"
+}
+
+reserve_node_address() {
+ local node_id="$1" existing host address
+ install -d -m 0700 "$MESH_DIR"
+ touch "$MESH_DIR/nodes.tsv"
+ chmod 0600 "$MESH_DIR/nodes.tsv"
+ existing="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$MESH_DIR/nodes.tsv")"
+ if [[ -n "$existing" ]]; then
+ printf '%s\n' "$existing"
+ return
+ fi
+ for host in $(seq 2 254); do
+ address="10.77.0.$host"
+ if ! awk -F '\t' -v address="$address" '$2 == address { found=1 } END { exit found ? 0 : 1 }' "$MESH_DIR/nodes.tsv"; then
+ printf '%s\t%s\t-\treserved\n' "$node_id" "$address" >>"$MESH_DIR/nodes.tsv"
+ printf '%s\n' "$address"
+ return
+ fi
+ done
+ fail "Mesh address pool is exhausted."
+}
+
+configure_node_wireguard() {
+ local node_id="$1" node_address="$2" hub_endpoint="$3" hub_public_key="$4" node_private node_public
+ ensure_mesh_packages
+ install -d -m 0700 "$WG_DIR" /etc/wireguard
+ if [[ ! -f "$WG_DIR/node.key" ]]; then
+ umask 077
+ wg genkey >"$WG_DIR/node.key"
+ fi
+ node_private="$(cat "$WG_DIR/node.key")"
+ node_public="$(printf '%s' "$node_private" | wg pubkey)"
+ printf '%s\n' "$node_public" >"$WG_DIR/node.pub"
+ cat >"/etc/wireguard/smm0.conf" <"$ext_file"
+ openssl x509 -req -sha256 -days 825 -in "$server_csr" -CA "$ca_cert" -CAkey "$ca_key" \
+ -CAserial "$serial_file" -CAcreateserial -out "$TEMP_DIR/control-server.crt" -extfile "$ext_file"
+ openssl pkcs12 -export -out "$ETC_DIR/control-ca.pfx" -inkey "$ca_key" -in "$ca_cert" -passout pass:
+ openssl pkcs12 -export -out "$ETC_DIR/control-server.pfx" -inkey "$server_key" \
+ -in "$TEMP_DIR/control-server.crt" -certfile "$ca_cert" -passout pass:
+ install -m 0644 "$ca_cert" "$ETC_DIR/control-ca.crt"
+ chown root:"$CONTROL_USER" "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx"
+ chmod 0640 "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx"
+}
+
+install_control() {
+ local archive="$1" public_host="$2" port="${3:-7443}" backup_id
+ require_root
+ validate_platform
+ validate_port "$port"
+ require_command openssl
+ require_command sha256sum
+ require_command tar
+ require_command systemctl
+ require_command sudo
+ require_command visudo
+ extract_archive "$archive"
+ [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing."
+ backup_id="$(create_backup control)"
+ ensure_system_user "$CONTROL_USER"
+ install -d -m 0750 -o root -g "$CONTROL_USER" "$ETC_DIR"
+ install -d -m 0750 -o "$CONTROL_USER" -g "$CONTROL_USER" "$STATE_DIR" "$STATE_DIR/backups"
+ install_tree_atomic "$TEMP_DIR/control" "$LIB_DIR/control" "root:root"
+ if [[ ! -f "$ETC_DIR/control-ca.pfx" || ! -f "$ETC_DIR/control-server.pfx" ]]; then
+ create_control_certificates "$public_host"
+ fi
+ cat >"$ETC_DIR/control.env" <"$ETC_DIR/control-public-url"
+ chown root:"$CONTROL_USER" "$ETC_DIR/control.env"
+ chmod 0640 "$ETC_DIR/control.env"
+ chmod 0644 "$ETC_DIR/control-public-url"
+ install -d -m 0755 "$(dirname "$POLICY_HELPER")"
+ install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER"
+ install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")"
+ install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND"
+ install -d -m 0755 "$LIB_DIR/bootstrap"
+ install -m 0644 "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$LIB_DIR/bootstrap/$FIREWALL_UNIT"
+ printf '%s\n' "$CONTROL_USER ALL=(root) NOPASSWD: $POLICY_HELPER *" >"$SUDOERS_FILE"
+ chmod 0440 "$SUDOERS_FILE"
+ visudo -cf "$SUDOERS_FILE" >/dev/null
+ install_unit "$TEMP_DIR/deploy/$CONTROL_UNIT" "$CONTROL_UNIT"
+ systemctl enable --now "$CONTROL_UNIT"
+ systemctl is-active --quiet "$CONTROL_UNIT" || {
+ systemctl status --no-pager "$CONTROL_UNIT" >&2 || true
+ fail "Control service failed; backup is $backup_id"
+ }
+ log "Control installed. Backup: $backup_id"
+ log "CA fingerprint: $(openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256 | cut -d= -f2)"
+}
+
+read_enrollment_token() {
+ if [[ -n "${SMM_ENROLL_TOKEN:-}" ]]; then
+ ENROLL_TOKEN="$SMM_ENROLL_TOKEN"
+ unset SMM_ENROLL_TOKEN
+ return
+ fi
+ [[ -t 0 ]] || fail "Set SMM_ENROLL_TOKEN or run from an interactive local terminal."
+ read -r -s -p "One-time enrollment token: " ENROLL_TOKEN
+ printf '\n'
+ [[ -n "$ENROLL_TOKEN" ]] || fail "Enrollment token is empty."
+}
+
+install_agent() {
+ local archive="$1" node_id="$2" control_url="$3" ca_cert="$4" backup_id
+ require_root
+ validate_platform
+ validate_node_id "$node_id"
+ validate_control_url "$control_url"
+ [[ -f "$ca_cert" ]] || fail "Control CA certificate not found: $ca_cert"
+ require_command sha256sum
+ require_command tar
+ require_command systemctl
+ require_command runuser
+ require_command openssl
+ openssl x509 -in "$ca_cert" -noout >/dev/null 2>&1 || fail "Invalid Control CA certificate."
+ extract_archive "$archive"
+ [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing."
+ [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] || fail "Provisioning helper binary is missing."
+ backup_id="$(create_backup agent)"
+ ensure_system_user "$AGENT_USER"
+ install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR"
+ install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent"
+ install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root"
+ install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root"
+ install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")"
+ install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND"
+ if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then
+ install -m 0600 -o "$AGENT_USER" -g "$AGENT_USER" "$ca_cert" "$ETC_DIR/control-ca.crt"
+ else
+ chown "$AGENT_USER:$AGENT_USER" "$ETC_DIR/control-ca.crt"
+ chmod 0600 "$ETC_DIR/control-ca.crt"
+ fi
+ cat >"$ETC_DIR/agent.env" <&2 || true
+ fail "Agent service failed; backup is $backup_id"
+ }
+ log "Agent $node_id installed and enrolled. Backup: $backup_id"
+}
+
+read_enrollment_code() {
+ if [[ -n "${SMM_ENROLL_CODE:-}" ]]; then
+ ENROLL_CODE="$SMM_ENROLL_CODE"
+ unset SMM_ENROLL_CODE
+ return
+ fi
+ [[ -t 0 ]] || fail "Set SMM_ENROLL_CODE or run from an interactive local terminal."
+ read -r -s -p "SMMNODE enrollment code: " ENROLL_CODE
+ printf '\n'
+ [[ -n "$ENROLL_CODE" ]] || fail "Enrollment code is empty."
+}
+
+confirm_ca_fingerprint() {
+ local ca_file="$1" answer
+ log "Control CA fingerprint: $(openssl x509 -in "$ca_file" -noout -fingerprint -sha256 | cut -d= -f2)"
+ if [[ "${SMM_ACCEPT_CA_FINGERPRINT:-}" == "1" ]]; then
+ return
+ fi
+ [[ -t 0 ]] || fail "Set SMM_ACCEPT_CA_FINGERPRINT=1 only after verifying the fingerprint out of band."
+ read -r -p "Type 'yes' after comparing this fingerprint with the Hub: " answer
+ [[ "$answer" == "yes" ]] || fail "Control CA fingerprint was not confirmed."
+}
+
+install_node_from_code() {
+ local archive="$1" prefix control_part ca_part node_part token_part
+ local endpoint_part hub_key_part address_part network_part extra
+ local control_url node_id token ca_file hub_endpoint hub_public_key node_address mesh_network
+ require_root
+ require_command base64
+ require_command openssl
+ read_enrollment_code
+ IFS='.' read -r prefix control_part ca_part node_part token_part endpoint_part \
+ hub_key_part address_part network_part extra <<<"$ENROLL_CODE"
+ ENROLL_CODE=""
+ [[ "$prefix" == "SMMNODE1" || "$prefix" == "SMMNODE2" ]] \
+ || fail "Unsupported SMMNODE enrollment code version."
+ [[ -n "$control_part" && -n "$ca_part" && -n "$node_part" && -n "$token_part" \
+ && -z "${extra:-}" ]] || fail "Invalid SMMNODE enrollment code."
+ if [[ "$prefix" == "SMMNODE1" ]]; then
+ [[ -z "${endpoint_part:-}${hub_key_part:-}${address_part:-}${network_part:-}" ]] \
+ || fail "Invalid SMMNODE1 enrollment code."
+ else
+ [[ -n "${endpoint_part:-}" && -n "${hub_key_part:-}" \
+ && -n "${address_part:-}" && -n "${network_part:-}" ]] \
+ || fail "Invalid SMMNODE2 mesh enrollment code."
+ fi
+ control_url="$(base64url_decode "$control_part")"
+ node_id="$(base64url_decode "$node_part")"
+ token="$(base64url_decode "$token_part")"
+ if [[ "$prefix" == "SMMNODE2" ]]; then
+ hub_endpoint="$(base64url_decode "$endpoint_part")"
+ hub_public_key="$(base64url_decode "$hub_key_part")"
+ node_address="$(base64url_decode "$address_part")"
+ mesh_network="$(base64url_decode "$network_part")"
+ [[ "$hub_endpoint" =~ ^[A-Za-z0-9.-]+:[0-9]{1,5}$ ]] || fail "Invalid Hub WireGuard endpoint."
+ validate_port "${hub_endpoint##*:}"
+ [[ "$hub_public_key" =~ ^[A-Za-z0-9+/]{43}=$ ]] || fail "Invalid Hub WireGuard public key."
+ [[ "$node_address" =~ ^10\.77\.0\.([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])$ ]] \
+ || fail "Invalid reserved mesh address."
+ [[ "$mesh_network" == "$MESH_NETWORK" ]] || fail "Unsupported mesh network."
+ fi
+ ca_file="$(mktemp -t smm-control-ca.XXXXXXXX.crt)"
+ chmod 0600 "$ca_file"
+ base64url_decode "$ca_part" >"$ca_file"
+ confirm_ca_fingerprint "$ca_file"
+ SMM_ENROLL_TOKEN="$token"
+ token=""
+ install_agent "$archive" "$node_id" "$control_url" "$ca_file"
+ rm -f -- "$ca_file"
+ if [[ "$prefix" == "SMMNODE2" ]]; then
+ configure_node_wireguard "$node_id" "$node_address" "$hub_endpoint" "$hub_public_key"
+ log "Node mesh configured. Copy this public peer code back to the Hub:"
+ printf '%s\n' "$MESH_PEER_CODE"
+ MESH_PEER_CODE=""
+ fi
+}
+
+update_role() {
+ local role="$1" archive="$2" binary unit user backup_id
+ require_root
+ validate_platform
+ extract_archive "$archive"
+ case "$role" in
+ control) binary="ochenstarik-smm-control"; unit="$CONTROL_UNIT"; user="root:root" ;;
+ agent) binary="ochenstarik-smm-agent"; unit="$AGENT_UNIT"; user="root:root" ;;
+ *) fail "Unknown role: $role" ;;
+ esac
+ [[ -x "$TEMP_DIR/$role/$binary" ]] || fail "$role binary is missing."
+ systemctl stop "$unit"
+ if [[ "$role" == "agent" ]]; then
+ systemctl stop "$PROVISIONING_HELPER_UNIT" 2>/dev/null || true
+ fi
+ backup_id="$(create_backup "$role")"
+ install_tree_atomic "$TEMP_DIR/$role" "$LIB_DIR/$role" "$user"
+ if [[ "$role" == "agent" ]]; then
+ [[ -x "$TEMP_DIR/provisioning-helper/ochenstarik-smm-provisioning-helper" ]] \
+ || fail "Provisioning helper binary is missing."
+ install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root"
+ install_unit "$TEMP_DIR/deploy/$AGENT_UNIT" "$AGENT_UNIT"
+ install_unit "$TEMP_DIR/deploy/$PROVISIONING_HELPER_UNIT" "$PROVISIONING_HELPER_UNIT"
+ systemctl enable --now "$PROVISIONING_HELPER_UNIT"
+ fi
+ systemctl start "$unit"
+ if ! systemctl is-active --quiet "$unit"; then
+ log "Update failed; restoring backup $backup_id"
+ restore_backup "$role" "$backup_id"
+ fail "$role update was rolled back."
+ fi
+ log "$role updated. Backup: $backup_id"
+}
+
+latest_backup_id() {
+ local role="$1" path
+ path="$(find "$BACKUP_DIR" -maxdepth 1 -type f \( -name "*-${role}-*.tar.gz" -o -name "*-${role}-*.empty" \) -printf '%f\n' 2>/dev/null | sort | tail -n1)"
+ [[ -n "$path" ]] || fail "No backup found for $role."
+ printf '%s\n' "${path%.tar.gz}" | sed 's/\.empty$//'
+}
+
+restore_backup() {
+ local role="$1" backup_id="$2" archive="$BACKUP_DIR/${backup_id}.tar.gz" unit
+ case "$role" in control) unit="$CONTROL_UNIT" ;; agent) unit="$AGENT_UNIT" ;; *) fail "Unknown role: $role" ;; esac
+ [[ -f "$archive" ]] || fail "Backup archive not found: $backup_id"
+ systemctl stop "$unit" || true
+ if [[ "$role" == "agent" ]]; then
+ systemctl stop "$PROVISIONING_HELPER_UNIT" || true
+ fi
+ tar -C / -xzf "$archive"
+ systemctl daemon-reload
+ if [[ "$role" == "agent" ]]; then
+ systemctl start "$PROVISIONING_HELPER_UNIT"
+ fi
+ systemctl start "$unit"
+ systemctl is-active --quiet "$unit" || fail "Rollback restored files but service is not active."
+ log "$role restored from $backup_id"
+}
+
+rollback_role() {
+ local role="$1" backup_id="${2:-}"
+ require_root
+ [[ -n "$backup_id" ]] || backup_id="$(latest_backup_id "$role")"
+ restore_backup "$role" "$backup_id"
+}
+
+show_status() {
+ local unit
+ for unit in "$CONTROL_UNIT" "$AGENT_UNIT"; do
+ if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then
+ printf '%s: %s\n' "$unit" "$(systemctl is-active "$unit" 2>/dev/null || true)"
+ else
+ printf '%s: not-installed\n' "$unit"
+ fi
+ done
+ if [[ -f "$ETC_DIR/control-ca.crt" ]] && command -v openssl >/dev/null; then
+ printf 'control-ca: %s\n' "$(openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256 | cut -d= -f2)"
+ fi
+}
+
+run_control_cli() {
+ local command_name="$1" identifier="$2"
+ require_root
+ validate_node_id "$identifier"
+ [[ -x "$LIB_DIR/control/ochenstarik-smm-control" ]] || fail "Control is not installed."
+ [[ -f "$ETC_DIR/control.env" ]] || fail "Control environment is missing."
+ require_command systemd-run
+ systemd-run --wait --pipe --quiet --collect \
+ --uid="$CONTROL_USER" \
+ --gid="$CONTROL_USER" \
+ -p "EnvironmentFile=$ETC_DIR/control.env" \
+ "$LIB_DIR/control/ochenstarik-smm-control" "$command_name" "$identifier"
+}
+
+create_node_code() {
+ local node_id="$1" token control_url ca_pem node_address hub_endpoint hub_public_key mesh_network
+ require_root
+ validate_node_id "$node_id"
+ [[ -r "$ETC_DIR/control-public-url" ]] || fail "Control public URL is missing; reinstall Control with PUBLIC_HOST."
+ [[ -r "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is missing."
+ require_command base64
+ control_url="$(tr -d '\r\n' <"$ETC_DIR/control-public-url")"
+ validate_control_url "$control_url"
+ token="$(run_control_cli token-create "$node_id")"
+ [[ -n "$token" && "$token" != *$'\n'* ]] || fail "Control returned an invalid enrollment token."
+ ca_pem="$(cat "$ETC_DIR/control-ca.crt")"
+ if [[ -r "$ETC_DIR/mesh.env" && -r "$WG_DIR/hub.pub" ]]; then
+ node_address="$(reserve_node_address "$node_id")"
+ hub_endpoint="$(read_mesh_value HUB_ENDPOINT)"
+ hub_public_key="$(read_mesh_value HUB_PUBLIC_KEY)"
+ mesh_network="$(read_mesh_value MESH_NETWORK)"
+ printf 'SMMNODE2.%s.%s.%s.%s.%s.%s.%s.%s\n' \
+ "$(printf '%s' "$control_url" | base64url_encode)" \
+ "$(printf '%s' "$ca_pem" | base64url_encode)" \
+ "$(printf '%s' "$node_id" | base64url_encode)" \
+ "$(printf '%s' "$token" | base64url_encode)" \
+ "$(printf '%s' "$hub_endpoint" | base64url_encode)" \
+ "$(printf '%s' "$hub_public_key" | base64url_encode)" \
+ "$(printf '%s' "$node_address" | base64url_encode)" \
+ "$(printf '%s' "$mesh_network" | base64url_encode)"
+ else
+ printf 'SMMNODE1.%s.%s.%s.%s\n' \
+ "$(printf '%s' "$control_url" | base64url_encode)" \
+ "$(printf '%s' "$ca_pem" | base64url_encode)" \
+ "$(printf '%s' "$node_id" | base64url_encode)" \
+ "$(printf '%s' "$token" | base64url_encode)"
+ fi
+ token=""
+}
+
+add_mesh_peer() {
+ local code="$1" prefix node_part address_part key_part extra
+ local node_id address public_key current tmp
+ require_root
+ require_command base64
+ require_command wg
+ [[ -r "$ETC_DIR/mesh.env" && -r "$MESH_DIR/nodes.tsv" ]] || fail "Mesh Hub is not initialized."
+ IFS='.' read -r prefix node_part address_part key_part extra <<<"$code"
+ [[ "$prefix" == "SMMPEER1" && -n "$node_part" && -n "$address_part" \
+ && -n "$key_part" && -z "${extra:-}" ]] || fail "Invalid SMMPEER1 code."
+ node_id="$(base64url_decode "$node_part")"
+ address="$(base64url_decode "$address_part")"
+ public_key="$(base64url_decode "$key_part")"
+ validate_node_id "$node_id"
+ [[ "$address" =~ ^10\.77\.0\.([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])$ ]] \
+ || fail "Invalid peer mesh address."
+ [[ "$public_key" =~ ^[A-Za-z0-9+/]{43}=$ ]] || fail "Invalid peer WireGuard public key."
+ current="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$MESH_DIR/nodes.tsv")"
+ [[ "$current" == "$address" ]] || fail "Peer address does not match the Hub reservation."
+ if awk -F '\t' -v address="$address" -v node="$node_id" '$2 == address && $1 != node { found=1 } END { exit found ? 0 : 1 }' "$MESH_DIR/nodes.tsv"; then
+ fail "Peer mesh address is already assigned."
+ fi
+ tmp="$(mktemp -p "$MESH_DIR" nodes.tsv.XXXXXXXX)"
+ awk -F '\t' -v OFS='\t' -v node="$node_id" -v address="$address" -v key="$public_key" \
+ '$1 == node { print node, address, key, "active"; found=1; next } { print } END { if (!found) exit 1 }' \
+ "$MESH_DIR/nodes.tsv" >"$tmp" || { rm -f -- "$tmp"; fail "Peer reservation is missing."; }
+ chmod 0600 "$tmp"
+ mv -- "$tmp" "$MESH_DIR/nodes.tsv"
+ render_hub_wireguard_config
+ systemctl restart wg-quick@smm0.service
+ systemctl is-active --quiet wg-quick@smm0.service || fail "WireGuard failed after peer registration."
+ log "Mesh peer $node_id activated at $address."
+}
+
+show_mesh_status() {
+ require_root
+ [[ -r "$ETC_DIR/mesh.env" ]] || fail "Mesh Hub is not initialized."
+ printf 'endpoint: %s\n' "$(read_mesh_value HUB_ENDPOINT)"
+ printf 'network: %s\n' "$(read_mesh_value MESH_NETWORK)"
+ if [[ -r "$MESH_DIR/nodes.tsv" ]]; then
+ printf '%-24s %-15s %-10s %s\n' NODE ADDRESS STATUS HANDSHAKE
+ while IFS=$'\t' read -r node_id address public_key status; do
+ [[ -n "$node_id" ]] || continue
+ local handshake="-"
+ if [[ "$status" == "active" ]]; then
+ handshake="$(wg show smm0 latest-handshakes 2>/dev/null | awk -v key="$public_key" '$1 == key { print $2; exit }')"
+ [[ -n "$handshake" && "$handshake" != "0" ]] || handshake="never"
+ fi
+ printf '%-24s %-15s %-10s %s\n' "$node_id" "$address" "$status" "$handshake"
+ done <"$MESH_DIR/nodes.tsv"
+ fi
+}
+
+show_ca_fingerprint() {
+ [[ -f "$ETC_DIR/control-ca.crt" ]] || fail "Control CA certificate is not installed."
+ require_command openssl
+ openssl x509 -in "$ETC_DIR/control-ca.crt" -noout -fingerprint -sha256
+}
+
+uninstall_agent() {
+ local purge="${1:-}"
+ require_root
+ systemctl disable --now "$AGENT_UNIT" 2>/dev/null || true
+ systemctl disable --now "$PROVISIONING_HELPER_UNIT" 2>/dev/null || true
+ rm -f -- "/etc/systemd/system/$AGENT_UNIT" "/etc/systemd/system/$PROVISIONING_HELPER_UNIT" "$ETC_DIR/agent.env"
+ rm -rf -- "$LIB_DIR/agent" "$LIB_DIR/provisioning-helper"
+ [[ "$purge" == "--purge" ]] && rm -rf -- "$STATE_DIR/agent" "$ETC_DIR/control-ca.crt"
+ systemctl daemon-reload
+ log "Agent removed${purge:+ ($purge)}."
+}
+
+uninstall_control() {
+ [[ "${1:-}" == "--confirm-destroy-control" ]] || fail "Control removal requires --confirm-destroy-control"
+ require_root
+ systemctl disable --now "$CONTROL_UNIT" 2>/dev/null || true
+ rm -f -- "/etc/systemd/system/$CONTROL_UNIT" "$ETC_DIR/control.env" \
+ "$ETC_DIR/control-ca.pfx" "$ETC_DIR/control-server.pfx" "$ETC_DIR/control-ca.crt" \
+ "$POLICY_HELPER" "$SUDOERS_FILE"
+ rm -rf -- "$LIB_DIR/control" "$STATE_DIR/control.db" "$STATE_DIR/control.db-wal" \
+ "$STATE_DIR/control.db-shm" "$STATE_DIR/backups"
+ systemctl daemon-reload
+ log "Control role and its state were removed."
+}
+
+preflight() {
+ validate_platform
+ local command_name
+ for command_name in openssl sha256sum tar systemctl getent useradd groupadd; do
+ require_command "$command_name"
+ done
+ log "Supported platform: $(. /etc/os-release; printf '%s %s' "$ID" "$VERSION_ID"), $(uname -m)"
+}
+
+main() {
+ local action="${1:-help}"
+ shift || true
+ case "$action" in
+ help|-h|--help) usage ;;
+ version|--version) printf '%s %s\n' "$PROGRAM" "$PROGRAM_VERSION" ;;
+ preflight) preflight ;;
+ verify-release) [[ $# -eq 1 ]] || fail "verify-release requires ARCHIVE"; verify_release_payload "$1" ;;
+ install-control) [[ $# -ge 2 && $# -le 3 ]] || fail "install-control requires ARCHIVE PUBLIC_HOST [HTTPS_PORT]"; install_control "$@" ;;
+ install-agent) [[ $# -eq 4 ]] || fail "install-agent requires ARCHIVE NODE_ID CONTROL_URL CA_CERT"; install_agent "$@" ;;
+ install-node) [[ $# -eq 1 ]] || fail "install-node requires ARCHIVE"; install_node_from_code "$1" ;;
+ mesh-init) [[ $# -ge 1 && $# -le 2 ]] || fail "mesh-init requires PUBLIC_ENDPOINT [WG_PORT]"; mesh_init "$@" ;;
+ peer-add) [[ $# -eq 1 ]] || fail "peer-add requires SMMPEER1_CODE"; add_mesh_peer "$1" ;;
+ mesh-status) [[ $# -eq 0 ]] || fail "mesh-status takes no arguments"; show_mesh_status ;;
+ update-control) [[ $# -eq 1 ]] || fail "update-control requires ARCHIVE"; update_role control "$1" ;;
+ update-agent) [[ $# -eq 1 ]] || fail "update-agent requires ARCHIVE"; update_role agent "$1" ;;
+ rollback) [[ $# -ge 1 && $# -le 2 ]] || fail "rollback requires control|agent [BACKUP_ID]"; rollback_role "$@" ;;
+ node-code) [[ $# -eq 1 ]] || fail "node-code requires NODE_ID"; create_node_code "$1" ;;
+ node-token) [[ $# -eq 1 ]] || fail "node-token requires NODE_ID"; run_control_cli token-create "$1" ;;
+ control-ca-fingerprint) [[ $# -eq 0 ]] || fail "control-ca-fingerprint takes no arguments"; show_ca_fingerprint ;;
+ status) show_status ;;
+ uninstall-agent) [[ $# -le 1 ]] || fail "uninstall-agent accepts only [--purge]"; uninstall_agent "${1:-}" ;;
+ uninstall-control) [[ $# -eq 1 ]] || fail "uninstall-control requires confirmation"; uninstall_control "$1" ;;
+ *) fail "Unknown action: $action (run with --help)" ;;
+ esac
+}
+
+main "$@"
diff --git a/deploy/ochenstarik-smm-agent.service b/deploy/ochenstarik-smm-agent.service
index 3e7659f..f82bbbf 100644
--- a/deploy/ochenstarik-smm-agent.service
+++ b/deploy/ochenstarik-smm-agent.service
@@ -1,13 +1,15 @@
[Unit]
Description=Ochenstarik Server Monitor Manager Agent
-After=network-online.target
+After=network-online.target ochenstarik-smm-provisioning-helper.service
Wants=network-online.target
+Requires=ochenstarik-smm-provisioning-helper.service
[Service]
Type=simple
User=ochenstarik-smm-agent
Group=ochenstarik-smm-agent
EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env
+WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/agent
ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/agent/ochenstarik-smm-agent
Restart=on-failure
RestartSec=10s
diff --git a/deploy/ochenstarik-smm-control.service b/deploy/ochenstarik-smm-control.service
index d4dac12..cd18b5a 100644
--- a/deploy/ochenstarik-smm-control.service
+++ b/deploy/ochenstarik-smm-control.service
@@ -8,6 +8,7 @@ Type=simple
User=ochenstarik-smm-control
Group=ochenstarik-smm-control
EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env
+WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/control
ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control
Restart=on-failure
RestartSec=10s
diff --git a/deploy/ochenstarik-smm-emergency b/deploy/ochenstarik-smm-emergency
new file mode 100755
index 0000000..51cbad9
--- /dev/null
+++ b/deploy/ochenstarik-smm-emergency
@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+readonly PROGRAM="ochenstarik-smm-emergency"
+readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager"
+readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager"
+readonly MARKER="$STATE_DIR/mesh/emergency-disabled"
+readonly CONTROL_UNIT="ochenstarik-smm-control.service"
+readonly AGENT_UNIT="ochenstarik-smm-agent.service"
+readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service"
+readonly WIREGUARD_UNIT="wg-quick@smm0.service"
+readonly NFT_FAMILY="inet"
+readonly NFT_TABLE="ochenstarik_smm"
+
+fail() { printf '%s: %s\n' "$PROGRAM" "$*" >&2; exit 78; }
+log() { printf '%s: %s\n' "$PROGRAM" "$*"; }
+
+usage() {
+ cat <<'EOF'
+Local Server Monitor Manager emergency recovery
+
+Usage:
+ ochenstarik-smm-emergency status
+ ochenstarik-smm-emergency mesh-disable
+ ochenstarik-smm-emergency mesh-enable
+ ochenstarik-smm-emergency firewall-restore
+
+Commands use only Server Monitor Manager-owned units, interface and nftables
+table. They do not modify the host SSH service or unrelated firewall rules.
+EOF
+}
+
+require_root() {
+ [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "this action must run as root (use sudo)"
+}
+
+unit_state() {
+ local unit="$1"
+ if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then
+ systemctl is-active "$unit" 2>/dev/null || true
+ else
+ printf '%s\n' "not-installed"
+ fi
+}
+
+show_status() {
+ local unit
+ for unit in "$CONTROL_UNIT" "$AGENT_UNIT" "$FIREWALL_UNIT" "$WIREGUARD_UNIT"; do
+ printf '%s: %s\n' "$unit" "$(unit_state "$unit")"
+ done
+ if command -v ip >/dev/null 2>&1 && ip link show smm0 >/dev/null 2>&1; then
+ printf '%s\n' "mesh-interface: present"
+ else
+ printf '%s\n' "mesh-interface: absent"
+ fi
+ if command -v nft >/dev/null 2>&1 && nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then
+ printf '%s\n' "mesh-firewall: loaded"
+ else
+ printf '%s\n' "mesh-firewall: absent"
+ fi
+ [[ -f "$MARKER" ]] && printf '%s\n' "emergency-lock: active" || printf '%s\n' "emergency-lock: inactive"
+ printf '%s\n' "backups:"
+ find "$STATE_DIR/bootstrap-backups" -maxdepth 1 -type f \
+ \( -name '*.tar.gz' -o -name '*.empty' \) -printf ' %f\n' 2>/dev/null | sort -r | head -n 10 || true
+}
+
+delete_project_firewall() {
+ if command -v nft >/dev/null 2>&1; then
+ nft delete table "$NFT_FAMILY" "$NFT_TABLE" 2>/dev/null || true
+ fi
+}
+
+mesh_disable() {
+ require_root
+ systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
+ systemctl disable --now "$FIREWALL_UNIT" 2>/dev/null || true
+ delete_project_firewall
+ install -d -m 0700 "$(dirname "$MARKER")"
+ printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$MARKER"
+ chmod 0600 "$MARKER"
+ log "Mesh disabled locally; Control and SSH were not changed."
+}
+
+restore_project_firewall() {
+ require_root
+ [[ -f "$ETC_DIR/mesh.nft" ]] || fail "managed firewall configuration is missing"
+ command -v nft >/dev/null 2>&1 || fail "nft is unavailable"
+ if ! nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then
+ nft --check -f "$ETC_DIR/mesh.nft"
+ fi
+ delete_project_firewall
+ if ! nft -f "$ETC_DIR/mesh.nft"; then
+ systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
+ fail "managed firewall restore failed; Mesh was disabled to fail closed"
+ fi
+ if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then
+ systemctl enable "$FIREWALL_UNIT" >/dev/null
+ fi
+ log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links."
+}
+
+mesh_enable() {
+ require_root
+ [[ -f /etc/wireguard/smm0.conf ]] || fail "WireGuard smm0 configuration is missing"
+ if [[ -f "$ETC_DIR/mesh.nft" ]]; then
+ restore_project_firewall
+ fi
+ systemctl enable "$WIREGUARD_UNIT" >/dev/null
+ systemctl restart "$WIREGUARD_UNIT"
+ rm -f -- "$MARKER"
+ log "Mesh enabled locally."
+}
+
+main() {
+ local action="${1:-help}"
+ shift || true
+ [[ $# -eq 0 ]] || fail "unexpected arguments"
+ case "$action" in
+ help|-h|--help) usage ;;
+ status) show_status ;;
+ mesh-disable) mesh_disable ;;
+ mesh-enable) mesh_enable ;;
+ firewall-restore) restore_project_firewall ;;
+ *) fail "unknown action: $action" ;;
+ esac
+}
+
+main "$@"
diff --git a/deploy/ochenstarik-smm-firewall.service b/deploy/ochenstarik-smm-firewall.service
new file mode 100644
index 0000000..f2d19e4
--- /dev/null
+++ b/deploy/ochenstarik-smm-firewall.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Server Monitor Manager Mesh firewall
+After=network-pre.target
+Before=wg-quick@smm0.service network-online.target
+Wants=network-pre.target
+
+[Service]
+Type=oneshot
+ExecStartPre=-/usr/sbin/nft delete table inet ochenstarik_smm
+ExecStart=/usr/sbin/nft -f /etc/ochenstarik-server-monitor-manager/mesh.nft
+ExecReload=/usr/sbin/nft -f /etc/ochenstarik-server-monitor-manager/mesh.nft
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/ochenstarik-smm-policy-apply b/deploy/ochenstarik-smm-policy-apply
new file mode 100755
index 0000000..040993e
--- /dev/null
+++ b/deploy/ochenstarik-smm-policy-apply
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+readonly TABLE_FAMILY="inet"
+readonly TABLE_NAME="ochenstarik_smm"
+readonly CHAIN_NAME="links"
+
+fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; }
+
+testing="${SMM_POLICY_TESTING:-0}"
+if [[ "$testing" != "1" ]]; then
+ [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required"
+ readonly STATE_FILE="/var/lib/ochenstarik-server-monitor-manager/mesh/nodes.tsv"
+else
+ readonly STATE_FILE="${SMM_POLICY_STATE_FILE:?SMM_POLICY_STATE_FILE is required in testing mode}"
+fi
+
+node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$'
+ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
+
+validate_node_id() {
+ [[ "$1" =~ $node_pattern ]] || fail "invalid node id"
+}
+
+lookup_node_ip() {
+ local node_id="$1" ip
+ [[ -r "$STATE_FILE" ]] || fail "mesh node state is unavailable"
+ ip="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$STATE_FILE")"
+ [[ "$ip" =~ $ipv4_pattern ]] || fail "node has no valid mesh address: $node_id"
+ printf '%s\n' "$ip"
+}
+
+validate_rule() {
+ local action="$1"
+ case "$action" in
+ link-connect) [[ $# -eq 6 ]] || fail "invalid link-connect argument count" ;;
+ link-disconnect) [[ $# -eq 5 ]] || fail "invalid link-disconnect argument count" ;;
+ *) fail "unsupported action" ;;
+ esac
+ validate_node_id "$2"
+ validate_node_id "$3"
+ [[ "$2" != "$3" ]] || fail "source and destination must differ"
+ [[ "$4" == "tcp" || "$4" == "udp" ]] || fail "invalid protocol"
+ [[ "$5" =~ ^[0-9]+$ ]] && (( 10#$5 >= 1 && 10#$5 <= 65535 )) || fail "invalid port"
+ if [[ "$action" == "link-connect" ]]; then
+ [[ "$6" =~ ^[0-9]+$ ]] && (( 10#$6 >= 0 && 10#$6 <= 525600 )) || fail "invalid TTL"
+ fi
+}
+
+run_nft() {
+ if [[ "$testing" == "1" ]]; then
+ printf 'nft'
+ printf ' %q' "$@"
+ printf '\n'
+ else
+ /usr/sbin/nft "$@"
+ fi
+}
+
+rule_exists() {
+ local comment="$1"
+ if [[ "$testing" == "1" ]]; then
+ return 1
+ fi
+ /usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \
+ | grep -Fq "comment \"$comment\""
+}
+
+connect_rule() {
+ local source_id="$1" target_id="$2" protocol="$3" port="$4"
+ local source_ip target_ip comment
+ source_ip="$(lookup_node_ip "$source_id")"
+ target_ip="$(lookup_node_ip "$target_id")"
+ comment="smm:${source_id}:${target_id}:${protocol}:${port}"
+ rule_exists "$comment" && return 0
+ run_nft add rule "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \
+ ip saddr "$source_ip" ip daddr "$target_ip" "$protocol" dport "$port" \
+ counter accept comment "$comment"
+}
+
+disconnect_rule() {
+ local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle
+ # Resolve both identities before touching firewall state.
+ lookup_node_ip "$source_id" >/dev/null
+ lookup_node_ip "$target_id" >/dev/null
+ comment="smm:${source_id}:${target_id}:${protocol}:${port}"
+ if [[ "$testing" == "1" ]]; then
+ printf 'nft-delete-comment %q\n' "$comment"
+ return
+ fi
+ while IFS= read -r handle; do
+ [[ "$handle" =~ ^[0-9]+$ ]] || continue
+ run_nft delete rule "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" handle "$handle"
+ done < <(
+ /usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" \
+ | grep -F "comment \"$comment\"" \
+ | sed -n 's/.* # handle \([0-9][0-9]*\)$/\1/p'
+ )
+}
+
+action="${1:-}"
+validate_rule "$@"
+case "$action" in
+ link-connect) connect_rule "$2" "$3" "$4" "$5" ;;
+ link-disconnect) disconnect_rule "$2" "$3" "$4" "$5" ;;
+esac
diff --git a/deploy/ochenstarik-smm-provisioning-helper.service b/deploy/ochenstarik-smm-provisioning-helper.service
new file mode 100644
index 0000000..ecf3697
--- /dev/null
+++ b/deploy/ochenstarik-smm-provisioning-helper.service
@@ -0,0 +1,31 @@
+[Unit]
+Description=Ochenstarik Server Monitor Manager restricted provisioning helper
+After=local-fs.target
+
+[Service]
+Type=simple
+User=root
+Group=ochenstarik-smm-agent
+UMask=0007
+RuntimeDirectory=ochenstarik-server-monitor-manager
+RuntimeDirectoryMode=0750
+WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper
+ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper/ochenstarik-smm-provisioning-helper
+Restart=on-failure
+RestartSec=5s
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectSystem=strict
+ProtectHome=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+LockPersonality=true
+RestrictSUIDSGID=true
+RestrictAddressFamilies=AF_UNIX
+CapabilityBoundingSet=
+AmbientCapabilities=
+
+[Install]
+WantedBy=multi-user.target
diff --git a/docs/installer-contract.md b/docs/installer-contract.md
index 29dfb3a..88981a5 100644
--- a/docs/installer-contract.md
+++ b/docs/installer-contract.md
@@ -82,11 +82,13 @@ control device-code DEVICE_ID
control node-code NODE_ID
control automation-token AUTOMATION_ID SOURCE_NODE_ID
emergency status
-emergency vpn-disable
-emergency ssh-restore BACKUP_ID
-emergency firewall-restore BACKUP_ID
+emergency mesh-disable
+emergency mesh-enable
+emergency firewall-restore
```
+Текущая emergency-команда покрывает уже реализованные Mesh/WireGuard и project-owned nftables resources. Действия `vpn-disable` и `ssh-restore BACKUP_ID` добавляются вместе с соответствующими provisioning-модулями и их root-only backup format; до появления этих модулей команда намеренно их отклоняет.
+
CLI является non-interactive, кроме локального ввода enrollment code и явных подтверждений опасного удаления. Машиночитаемый режим возвращает versioned JSON и стабильные exit codes.
## 6. Идемпотентность и обновление
diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md
new file mode 100644
index 0000000..13e73c3
--- /dev/null
+++ b/docs/linux-bootstrap.md
@@ -0,0 +1,111 @@
+# Linux bootstrap и собственная Mesh-сеть
+
+Server Monitor Manager устанавливает Control (Hub) и Agent (Node) из проверяемого release-архива. Для связи серверов используется собственная WireGuard-сеть `10.77.0.0/24`; Tailscale и другие внешние VPN-сервисы не требуются.
+
+Текущая версия предназначена для alpha-тестирования на Ubuntu Server 22.04/24.04 и Debian 12/13 (`amd64`, `arm64`, systemd). Hub должен иметь публичный IPv4-адрес или DNS-имя и доступный UDP-порт. Node может находиться за NAT без белого IP.
+
+## Файлы релиза
+
+Скачайте из одного GitHub Release:
+
+- `ochenstarik-server-monitor-manager.sh` и `.sha256`;
+- `server-monitor-manager-linux-x64.tar.gz` или `server-monitor-manager-linux-arm64.tar.gz`;
+- соответствующий `.tar.gz.sha256`.
+
+Bootstrap проверяет SHA-256 до распаковки и принимает в архиве только каталоги `agent`, `control`, `deploy` и `bootstrap`.
+
+## 1. Установка главного сервера (Hub)
+
+```bash
+sha256sum -c ochenstarik-server-monitor-manager.sh.sha256
+chmod 700 ochenstarik-server-monitor-manager.sh
+sudo ./ochenstarik-server-monitor-manager.sh preflight
+sudo ./ochenstarik-server-monitor-manager.sh verify-release \
+ ./server-monitor-manager-linux-x64.tar.gz
+sudo ./ochenstarik-server-monitor-manager.sh install-control \
+ ./server-monitor-manager-linux-x64.tar.gz \
+ hub.example.com \
+ 7443
+sudo ./ochenstarik-server-monitor-manager.sh mesh-init hub.example.com 51820
+```
+
+Откройте на Hub и во внешнем firewall/security group:
+
+- TCP `7443` для Control HTTPS;
+- UDP `51820` для WireGuard.
+
+`install-control` создаёт локальный Control CA и HTTPS-сертификат. Приватный ключ CA не включается в коды подключения и остаётся на Hub. `mesh-init` устанавливает `wireguard-tools`, `nftables` и `iproute2`, создаёт ключ Hub, интерфейс `smm0`, включает IPv4 forwarding и межсерверный firewall с запретом по умолчанию.
+
+## 2. Выпуск кода для Node
+
+```bash
+sudo ./ochenstarik-server-monitor-manager.sh node-code home
+sudo ./ochenstarik-server-monitor-manager.sh control-ca-fingerprint
+```
+
+После `mesh-init` команда выдаёт одноразовый код `SMMNODE2`. Он содержит Control URL, публичный сертификат CA, Node ID, десятиминутный enrollment token, endpoint и публичный ключ Hub, а также зарезервированный Mesh-адрес. Обращайтесь с кодом как с временным секретом.
+
+## 3. Установка вторичного сервера (Node)
+
+Node может быть за NAT. Ему нужен исходящий доступ к TCP-порту Control и UDP-порту WireGuard на Hub.
+
+```bash
+sudo ./ochenstarik-server-monitor-manager.sh install-node \
+ ./server-monitor-manager-linux-x64.tar.gz
+```
+
+Вставьте `SMMNODE2` в скрытый prompt. Сверьте показанный SHA-256 fingerprint CA с Hub и введите `yes`. После mTLS enrollment установщик создаст локальный приватный ключ WireGuard, запустит `smm0` с `PersistentKeepalive = 25` и выведет публичный код `SMMPEER1`.
+
+Для автоматизированного стенда допускается передача кода только в окружении процесса после отдельной сверки fingerprint:
+
+```bash
+sudo SMM_ENROLL_CODE='SMMNODE2....' SMM_ACCEPT_CA_FINGERPRINT=1 \
+ ./ochenstarik-server-monitor-manager.sh install-node \
+ ./server-monitor-manager-linux-x64.tar.gz
+```
+
+## 4. Активация Node на Hub
+
+Скопируйте выведенный Node код `SMMPEER1` на Hub:
+
+```bash
+sudo ./ochenstarik-server-monitor-manager.sh peer-add 'SMMPEER1....'
+sudo ./ochenstarik-server-monitor-manager.sh mesh-status
+```
+
+Hub проверяет Node ID и ранее зарезервированный IP, сохраняет публичный ключ peer и перезапускает интерфейс. Приватный ключ Node никогда не покидает Node.
+
+## Изоляция и управляемые соединения
+
+Трафик `smm0 -> smm0` по умолчанию блокируется. Control вызывает root-helper только для точных правил `source IP -> target IP`, протокола и порта. Поддерживаются команды helper `link-connect SOURCE TARGET tcp|udp PORT TTL_MINUTES` и `link-disconnect SOURCE TARGET tcp|udp PORT`. Это позволяет вручную подключать AI-агент к выбранному серверу и затем отзывать доступ, не открывая связь между всеми Node.
+
+В текущем alpha TTL валидируется и хранится Control, а удаление просроченных правил зависит от reconciliation Control. После перезапуска firewall разрешающие правила должны быть повторно применены Control.
+
+## Обслуживание
+
+```bash
+sudo ./ochenstarik-server-monitor-manager.sh status
+sudo ./ochenstarik-server-monitor-manager.sh mesh-status
+sudo ./ochenstarik-server-monitor-manager.sh update-control ARCHIVE
+sudo ./ochenstarik-server-monitor-manager.sh update-agent ARCHIVE
+sudo ./ochenstarik-server-monitor-manager.sh rollback control
+sudo ./ochenstarik-server-monitor-manager.sh rollback agent
+sudo ./ochenstarik-server-monitor-manager.sh uninstall-agent
+sudo ./ochenstarik-server-monitor-manager.sh uninstall-agent --purge
+sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy-control
+```
+
+Update создаёт root-only backup перед заменой binaries и автоматически восстанавливает предыдущую версию, если сервис не запускается. Перед alpha-тестом на реальных серверах обязательно сохраните отдельную консольную/SSH-сессию и не закрывайте основной административный доступ firewall-правилами проекта.
+
+## Локальное аварийное восстановление
+
+Установщик размещает независимую от Control Hub команду `/usr/local/sbin/ochenstarik-smm-emergency`. Она принимает только фиксированные действия и управляет исключительно интерфейсом `smm0`, systemd units и nftables-таблицей Server Monitor Manager:
+
+```bash
+sudo ochenstarik-smm-emergency status
+sudo ochenstarik-smm-emergency mesh-disable
+sudo ochenstarik-smm-emergency firewall-restore
+sudo ochenstarik-smm-emergency mesh-enable
+```
+
+`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub.
diff --git a/docs/linux-platform-matrix.md b/docs/linux-platform-matrix.md
new file mode 100644
index 0000000..9f1db6d
--- /dev/null
+++ b/docs/linux-platform-matrix.md
@@ -0,0 +1,20 @@
+# Linux platform matrix
+
+Workflow `linux-platform-matrix.yml` собирает те же self-contained release-архивы, которые устанавливает production bootstrap, отдельно для `linux-x64` и `linux-arm64`.
+
+## Реальные Ubuntu VM
+
+На GitHub-hosted VM проверяются:
+
+- Ubuntu 22.04 x64;
+- Ubuntu 24.04 x64;
+- Ubuntu 22.04 arm64;
+- Ubuntu 24.04 arm64.
+
+Каждая VM выполняет `preflight`, проверяет checksum и содержимое архива, дважды устанавливает Control, проверяет emergency-команду, перезапускает systemd service и обращается к HTTPS `/healthz` через созданный CA.
+
+## Debian systemd containers
+
+Debian 12/13 проверяются для x64 и arm64 в privileged systemd-контейнерах на соответствующей архитектуре runner. Тест выполняет чистую и повторную установку, затем полностью перезапускает контейнер и проверяет автоматический запуск Control и HTTPS healthcheck после нового systemd boot.
+
+Systemd-контейнер проверяет дистрибутив, users, permissions, units, certificates, SQLite state и service lifecycle, но не считается полноценной Debian VM. Отдельным незакрытым критерием остаётся reboot настоящих Debian VM с собственным kernel. Физическая проверка WireGuard/nftables также выполняется по `three-server-acceptance.md`, поскольку GitHub runners не моделируют внешний NAT и cloud firewall.
diff --git a/docs/provisioning-vpn-requirements.md b/docs/provisioning-vpn-requirements.md
index f53132f..92ebb30 100644
--- a/docs/provisioning-vpn-requirements.md
+++ b/docs/provisioning-vpn-requirements.md
@@ -278,6 +278,7 @@ GET /api/v1/nodes/{nodeId}/configuration
POST /api/v1/nodes/{nodeId}/provisioning/preflight
POST /api/v1/nodes/{nodeId}/provisioning/jobs
GET /api/v1/provisioning/jobs/{jobId}
+GET /api/v1/provisioning/jobs/{jobId}/plan
POST /api/v1/provisioning/jobs/{jobId}/confirm
POST /api/v1/provisioning/jobs/{jobId}/cancel
POST /api/v1/provisioning/jobs/{jobId}/rollback
@@ -288,6 +289,8 @@ GET /api/v1/nodes/{nodeId}/vpn-profiles
Agent получает только задания своего Node. Automation identity не создаёт provisioning jobs, не читает VPN secrets и не управляет пользователями. Mutation требует Operator certificate, idempotency key и audit reason.
+Подтверждённая опасная операция дополнительно требует короткоживущий execution grant. Control подписывает его ECDSA-ключом Control CA и связывает с `node_id`, `job_id`, action/schema, SHA-256 подтверждённого plan, nonce и сроком действия. Agent получает grant через `POST /api/v1/agents/provisioning/jobs/{jobId}/execution-grant`; root-helper проверяет подпись по закреплённому Control CA до запуска любой мутации.
+
Каждый action type имеет отдельную versioned JSON Schema. Неизвестные смысловые поля отклоняются на Control, Agent и helper.
## 15. Хранение и аудит
diff --git a/docs/roadmap.md b/docs/roadmap.md
index 76d60d7..4fe63e5 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -2,6 +2,8 @@
Подробные требования к собственному bootstrap, управляемой настройке Linux и Xray находятся в [ТЗ Provisioning и Xray VPN](provisioning-vpn-requirements.md). Все компоненты Server Monitor Manager разрабатываются, версионируются и выпускаются только в этом репозитории.
+Текущее автоматическое покрытие поддерживаемых Linux-платформ описано в [Linux platform matrix](linux-platform-matrix.md).
+
## Этап 0 — граница и базовая архитектура
- [x] переименовать проект в Server Monitor Manager;
@@ -90,33 +92,43 @@
## Этап 7 — собственный bootstrap Server Monitor Manager
-- [ ] добавить bootstrap source в этот репозиторий;
-- [ ] публиковать bootstrap, checksum и signed compatibility manifest в release;
-- [ ] поддержать Ubuntu 22.04/24.04 и Debian 12/13, `amd64`/`arm64`;
-- [ ] добавить non-interactive install/update/rollback/uninstall;
-- [ ] устанавливать Agent, restricted helper и systemd units;
-- [ ] локально создавать Node keys/CSR и погашать bootstrap enrollment;
-- [ ] добавить VM CI matrix, повторную установку и reboot;
-- [ ] добавить локальную emergency recovery command.
+- [x] добавить bootstrap source в этот репозиторий;
+- [x] упаковывать bootstrap, checksum и compatibility manifest в release workflow;
+- [ ] добавить криптографическую подпись compatibility manifest для production release;
+- [x] проверять Ubuntu 22.04/24.04 и Debian 12/13, `amd64`/`arm64`;
+- [x] добавить non-interactive install/update/rollback/uninstall;
+- [x] устанавливать Control/Agent, restricted helper и systemd units;
+- [x] локально создавать Agent key/CSR, выполнять mTLS enrollment и не сохранять token;
+- [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов;
+- [x] реализовать nftables policy helper вместо временного deny-by-default helper;
+- [x] добавить native Ubuntu 22.04/24.04 x64/arm64 VM CI и повторную установку;
+- [x] добавить Debian 12/13 x64/arm64 systemd-container restart matrix;
+- [ ] добавить полный reboot настоящих Debian VM;
+- [x] добавить локальную emergency recovery command для текущих Mesh/firewall-компонентов.
## Этап 8 — Provisioning control plane
-- [ ] модели и SQLite migrations для ProvisioningJob;
-- [ ] state machine, confirmations, cancellation, retry и rollback;
-- [ ] обязательные idempotency key, audit reason и job TTL;
-- [ ] Agent job channel только для собственного `node_id`;
-- [ ] versioned JSON schemas для каждого action type;
-- [ ] restricted root helper через Unix socket;
-- [ ] structured redacted events и progress;
-- [ ] `NeedsReconciliation` после неопределённого результата;
-- [ ] desired/factual configuration и drift;
-- [ ] запрет параллельных несовместимых опасных заданий.
+- [x] модели и SQLite migration v2 для ProvisioningJob;
+- [x] state machine, confirmations, cancellation, retry и rollback;
+- [x] создание, чтение, подтверждение и отмена через Operator API;
+- [ ] выполнение, retry, verification и rollback в полной state machine;
+- [x] обязательные idempotency key, audit reason и job TTL;
+- [x] атомарный Agent job channel только для собственного `node_id`;
+- [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`;
+- [ ] versioned JSON schemas для остальных action type;
+- [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`);
+- [x] двухфазный `system.base-install`: сохранённый проверенный plan до Operator confirmation;
+- [x] короткоживущий ECDSA execution grant, привязанный к Node, job и SHA-256 подтверждённого plan;
+- [x] structured redacted events, bounded Operator history и progress;
+- [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата;
+- [ ] desired/factual configuration и drift (`preflight` завершён; остальные action type ещё не подключены);
+- [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант).
## Этап 9 — базовая настройка и пользователи
- [ ] preflight ОС, архитектуры, SSH, firewall, APT и capabilities;
- [ ] Desktop wizard timezone/locale/packages/swap/unattended upgrades;
-- [ ] versioned package allowlist;
+- [x] versioned package allowlist (catalog v1 с фиксированными package groups);
- [ ] root-only backups и symlink protection;
- [ ] user lifecycle без sudo по умолчанию;
- [ ] SSH public keys, fingerprints и permissions;
diff --git a/src/ServerMonitorManager.Agent/AgentClient.cs b/src/ServerMonitorManager.Agent/AgentClient.cs
index 013ddf6..0a0e1d2 100644
--- a/src/ServerMonitorManager.Agent/AgentClient.cs
+++ b/src/ServerMonitorManager.Agent/AgentClient.cs
@@ -2,6 +2,7 @@ using System.Net;
using System.Net.Http.Json;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
+using System.Text;
using ServerMonitorManager.Core;
namespace ServerMonitorManager.Agent;
@@ -105,12 +106,161 @@ internal sealed class AgentClient(AgentOptions options)
}
}
+ try
+ {
+ await ExecuteNextProvisioningJobAsync(client, cancellationToken);
+ }
+ catch (Exception exception) when (exception is not OperationCanceledException)
+ {
+ Console.Error.WriteLine($"Provisioning poll failed: {exception.Message}");
+ }
+
var elapsed = DateTimeOffset.UtcNow - iterationStartedAt;
var remaining = collectionDelay - elapsed;
await Task.Delay(remaining > TimeSpan.Zero ? remaining : TimeSpan.FromSeconds(1), cancellationToken);
}
}
+ private async Task ExecuteNextProvisioningJobAsync(
+ HttpClient client,
+ CancellationToken cancellationToken)
+ {
+ using var response = await client.GetAsync("api/v1/agents/provisioning/jobs/next", cancellationToken);
+ if (response.StatusCode == HttpStatusCode.NoContent)
+ {
+ return;
+ }
+ response.EnsureSuccessStatusCode();
+ var job = await response.Content.ReadFromJsonAsync(
+ SmmJsonContext.Default.ProvisioningJob,
+ cancellationToken)
+ ?? throw new InvalidOperationException("Control service returned an empty provisioning job.");
+
+ if (job.SchemaVersion != 1)
+ {
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
+ "dispatch", "action.unsupported", "Unsupported provisioning action.", cancellationToken);
+ return;
+ }
+
+ if (job.ActionType == "system.base-install"
+ && job.State == ProvisioningJobStates.Preflight)
+ {
+ try
+ {
+ var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath);
+ var plan = await helper.CreateBaseInstallPlanAsync(job, cancellationToken);
+ await ReportBaseInstallPlanAsync(client, job, plan, cancellationToken);
+ Console.WriteLine($"Base installation plan {job.Id} is awaiting confirmation.");
+ }
+ catch (Exception exception) when (exception is not OperationCanceledException)
+ {
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
+ "preflight", "base-install.plan-failed",
+ "Base installation plan generation failed.", cancellationToken);
+ Console.Error.WriteLine($"Base installation plan {job.Id} failed: {exception.Message}");
+ }
+ return;
+ }
+
+ if (job.ActionType != "preflight")
+ {
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
+ "dispatch", "action.unsupported", "Unsupported provisioning action.", cancellationToken);
+ return;
+ }
+
+ ProvisioningPreflightResult result;
+ try
+ {
+ var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath);
+ result = await helper.RunPreflightAsync(job, cancellationToken);
+ }
+ catch (Exception exception) when (exception is not OperationCanceledException)
+ {
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
+ "preflight", "preflight.failed", "Preflight helper failed.", cancellationToken);
+ Console.Error.WriteLine($"Preflight {job.Id} failed: {exception.Message}");
+ return;
+ }
+
+ await ReportPreflightFactsAsync(client, job, result, cancellationToken);
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Running, 40,
+ "inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken);
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Verifying, 80,
+ "verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken);
+ await ReportProvisioningAsync(
+ client, job, ProvisioningJobStates.Completed, 100,
+ "completed", "preflight.completed", "Preflight completed.", cancellationToken);
+ Console.WriteLine(
+ $"Preflight {job.Id} completed: {result.OperatingSystem} "
+ + $"{result.OperatingSystemVersion} {result.Architecture}.");
+ }
+
+ private static async Task ReportProvisioningAsync(
+ HttpClient client,
+ ProvisioningJob job,
+ string state,
+ int progress,
+ string step,
+ string eventCode,
+ string message,
+ CancellationToken cancellationToken)
+ {
+ var request = new ProvisioningJobProgressRequest(
+ state, progress, step, eventCode, message, CreateOperationId(job.Id, state));
+ using var response = await client.PostAsJsonAsync(
+ $"api/v1/agents/provisioning/jobs/{job.Id}/progress",
+ request,
+ SmmJsonContext.Default.ProvisioningJobProgressRequest,
+ cancellationToken);
+ response.EnsureSuccessStatusCode();
+ }
+
+ private static async Task ReportPreflightFactsAsync(
+ HttpClient client,
+ ProvisioningJob job,
+ ProvisioningPreflightResult facts,
+ CancellationToken cancellationToken)
+ {
+ var request = new ProvisioningPreflightReportRequest(
+ facts, DateTimeOffset.UtcNow, CreateOperationId(job.Id, "facts"));
+ using var response = await client.PostAsJsonAsync(
+ $"api/v1/agents/provisioning/jobs/{job.Id}/preflight-facts",
+ request,
+ SmmJsonContext.Default.ProvisioningPreflightReportRequest,
+ cancellationToken);
+ response.EnsureSuccessStatusCode();
+ }
+
+ private static async Task ReportBaseInstallPlanAsync(
+ HttpClient client,
+ ProvisioningJob job,
+ SystemBaseInstallPlan plan,
+ CancellationToken cancellationToken)
+ {
+ var request = new SystemBaseInstallPlanReportRequest(
+ plan, CreateOperationId(job.Id, "base-install-plan"));
+ using var response = await client.PostAsJsonAsync(
+ $"api/v1/agents/provisioning/jobs/{job.Id}/base-install-plan",
+ request,
+ SmmJsonContext.Default.SystemBaseInstallPlanReportRequest,
+ cancellationToken);
+ response.EnsureSuccessStatusCode();
+ }
+
+ private static string CreateOperationId(string jobId, string state)
+ {
+ var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}"));
+ return new Guid(digest.AsSpan(0, 16)).ToString();
+ }
+
private static async Task SendHeartbeatAsync(
HttpClient client,
AgentHeartbeat heartbeat,
diff --git a/src/ServerMonitorManager.Agent/AgentOptions.cs b/src/ServerMonitorManager.Agent/AgentOptions.cs
index 68795da..b5603d5 100644
--- a/src/ServerMonitorManager.Agent/AgentOptions.cs
+++ b/src/ServerMonitorManager.Agent/AgentOptions.cs
@@ -6,6 +6,7 @@ public sealed class AgentOptions
public Uri ControlUrl { get; init; } = new("https://127.0.0.1:7443");
public string StateDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/agent";
public string CertificateAuthorityPath { get; init; } = "/etc/ochenstarik-server-monitor-manager/control-ca.crt";
+ public string ProvisioningSocketPath { get; init; } = "/run/ochenstarik-server-monitor-manager/provisioning.sock";
public int HeartbeatSeconds { get; init; } = 30;
public int BufferMaxSamples { get; init; } = 720;
public int BufferRecentSamples { get; init; } = 120;
diff --git a/src/ServerMonitorManager.Agent/Program.cs b/src/ServerMonitorManager.Agent/Program.cs
index 7e536fa..c9693a7 100644
--- a/src/ServerMonitorManager.Agent/Program.cs
+++ b/src/ServerMonitorManager.Agent/Program.cs
@@ -19,7 +19,8 @@ if (options.HeartbeatSeconds is < 10 or > 300
|| options.BufferRecentSamples >= options.BufferMaxSamples
|| options.BufferDownsampleFactor is < 2 or > 100
|| options.UploadBatchSize is < 1 or > 100
- || options.MaxRetrySeconds is < 10 or > 3600)
+ || options.MaxRetrySeconds is < 10 or > 3600
+ || !Path.IsPathFullyQualified(options.ProvisioningSocketPath))
{
Console.Error.WriteLine(
"Invalid buffer settings: heartbeat 10-300s, max samples 10-10000, recent samples below max, "
diff --git a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs
new file mode 100644
index 0000000..d0c2673
--- /dev/null
+++ b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs
@@ -0,0 +1,76 @@
+using System.Net.Sockets;
+using System.Text;
+using System.Text.Json;
+using ServerMonitorManager.Core;
+
+namespace ServerMonitorManager.Agent;
+
+public sealed class ProvisioningHelperClient(string socketPath)
+{
+ private const int MaximumResponseBytes = 16 * 1024;
+
+ public async Task RunPreflightAsync(
+ ProvisioningJob job,
+ CancellationToken cancellationToken)
+ {
+ var request = new ProvisioningHelperRequest(
+ "1", job.Id, job.ActionType, job.SchemaVersion,
+ ProvisioningActionCatalog.PreflightModuleHash, job.Parameters);
+ var response = await SendAsync(request, cancellationToken);
+ if (!response.Success || response.Preflight is null)
+ {
+ throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}");
+ }
+ return response.Preflight;
+ }
+
+ public async Task CreateBaseInstallPlanAsync(
+ ProvisioningJob job,
+ CancellationToken cancellationToken)
+ {
+ var request = new ProvisioningHelperRequest(
+ "1", job.Id, job.ActionType, job.SchemaVersion,
+ ProvisioningActionCatalog.SystemBaseInstallModuleHash, job.Parameters);
+ var response = await SendAsync(request, cancellationToken);
+ if (!response.Success || response.BaseInstallPlan is null)
+ {
+ throw new InvalidOperationException($"Provisioning helper rejected the request: {response.Code}");
+ }
+ return response.BaseInstallPlan;
+ }
+
+ private async Task SendAsync(
+ ProvisioningHelperRequest request,
+ CancellationToken cancellationToken)
+ {
+ using var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified);
+ await socket.ConnectAsync(new UnixDomainSocketEndPoint(socketPath), cancellationToken);
+ await using var stream = new NetworkStream(socket, ownsSocket: false);
+ var json = JsonSerializer.Serialize(request, SmmJsonContext.Default.ProvisioningHelperRequest) + "\n";
+ await stream.WriteAsync(Encoding.UTF8.GetBytes(json), cancellationToken);
+ var payload = await ReadResponseAsync(stream, cancellationToken);
+ var response = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperResponse)
+ ?? throw new InvalidDataException("Provisioning helper returned an empty response.");
+ return response;
+ }
+
+ private static async Task ReadResponseAsync(Stream stream, CancellationToken cancellationToken)
+ {
+ using var buffer = new MemoryStream();
+ var singleByte = new byte[1];
+ while (buffer.Length <= MaximumResponseBytes)
+ {
+ var count = await stream.ReadAsync(singleByte, cancellationToken);
+ if (count == 0 || singleByte[0] == (byte)'\n')
+ {
+ break;
+ }
+ buffer.WriteByte(singleByte[0]);
+ }
+ if (buffer.Length == 0 || buffer.Length > MaximumResponseBytes)
+ {
+ throw new InvalidDataException("Provisioning helper response size is invalid.");
+ }
+ return buffer.ToArray();
+ }
+}
diff --git a/src/ServerMonitorManager.Control/CertificateAuthority.cs b/src/ServerMonitorManager.Control/CertificateAuthority.cs
index 6ec9a63..91b0796 100644
--- a/src/ServerMonitorManager.Control/CertificateAuthority.cs
+++ b/src/ServerMonitorManager.Control/CertificateAuthority.cs
@@ -1,6 +1,7 @@
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Options;
+using ServerMonitorManager.Core;
namespace ServerMonitorManager.Control;
@@ -65,6 +66,49 @@ public sealed class CertificateAuthority : IDisposable
notAfter);
}
+ public ProvisioningExecutionGrant SignProvisioningExecutionGrant(
+ ProvisioningJob job,
+ SystemBaseInstallPlan plan,
+ DateTimeOffset issuedAt,
+ TimeSpan lifetime)
+ {
+ if (job.ActionType != "system.base-install"
+ || job.SchemaVersion != 1
+ || job.State != ProvisioningJobStates.Queued
+ || !job.ConfirmationRequired
+ || job.ConfirmedAt is null
+ || lifetime <= TimeSpan.Zero
+ || lifetime > ProvisioningExecutionGrantCodec.MaximumLifetime)
+ {
+ throw new InvalidOperationException(
+ "Only a confirmed queued base installation job can receive an execution grant.");
+ }
+
+ var grant = new ProvisioningExecutionGrant(
+ ProvisioningExecutionGrantCodec.ProtocolVersion,
+ job.Id,
+ job.NodeId,
+ job.ActionType,
+ job.SchemaVersion,
+ ProvisioningExecutionGrantCodec.ComputePlanSha256(plan),
+ issuedAt.ToUnixTimeSeconds(),
+ issuedAt.Add(lifetime).ToUnixTimeSeconds(),
+ Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)),
+ ProvisioningExecutionGrantCodec.SignatureAlgorithm,
+ string.Empty);
+ using var key = _issuer.GetECDsaPrivateKey();
+ if (key is not { KeySize: 256 })
+ {
+ throw new InvalidOperationException(
+ "Control CA must use an ECDSA P-256 key to sign provisioning execution grants.");
+ }
+ var signature = key.SignData(
+ ProvisioningExecutionGrantCodec.CreateSigningPayload(grant),
+ HashAlgorithmName.SHA256,
+ DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
+ return grant with { Signature = ProvisioningExecutionGrantCodec.EncodeBase64Url(signature) };
+ }
+
public void Dispose() => _issuer.Dispose();
}
diff --git a/src/ServerMonitorManager.Control/ControlMaintenance.cs b/src/ServerMonitorManager.Control/ControlMaintenance.cs
index 10285c1..de3db45 100644
--- a/src/ServerMonitorManager.Control/ControlMaintenance.cs
+++ b/src/ServerMonitorManager.Control/ControlMaintenance.cs
@@ -10,7 +10,9 @@ public sealed record ControlMaintenanceResult(
int MetricsDeleted,
int IdempotencyDeleted,
int AuditDeleted,
- int TokensDeleted);
+ int TokensDeleted,
+ int ProvisioningJobsCancelled,
+ int ProvisioningJobsNeedingReconciliation);
public sealed class LinkExpirationBackgroundService(
LinkService links,
@@ -65,15 +67,20 @@ public sealed class ControlMaintenanceBackgroundService(
{
var result = await store.MaintainAsync(timeProvider.GetUtcNow(), stoppingToken);
await backups.CreateIfDueAsync(timeProvider.GetUtcNow(), stoppingToken);
- if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted + result.TokensDeleted > 0)
+ if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted
+ + result.TokensDeleted + result.ProvisioningJobsCancelled
+ + result.ProvisioningJobsNeedingReconciliation > 0)
{
logger.LogInformation(
"Control maintenance removed {Metrics} metrics, {Idempotency} replay records, "
- + "{Audit} audit records, and {Tokens} enrollment tokens.",
+ + "{Audit} audit records, and {Tokens} enrollment tokens; cancelled {Cancelled} "
+ + "expired jobs and marked {Reconciliation} jobs for reconciliation.",
result.MetricsDeleted,
result.IdempotencyDeleted,
result.AuditDeleted,
- result.TokensDeleted);
+ result.TokensDeleted,
+ result.ProvisioningJobsCancelled,
+ result.ProvisioningJobsNeedingReconciliation);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs
index d56a832..469294a 100644
--- a/src/ServerMonitorManager.Control/ControlStore.cs
+++ b/src/ServerMonitorManager.Control/ControlStore.cs
@@ -8,9 +8,9 @@ using ServerMonitorManager.Core;
namespace ServerMonitorManager.Control;
-public sealed class ControlStore(IOptions options)
+public sealed partial class ControlStore(IOptions options)
{
- private const int CurrentSchemaVersion = 1;
+ private const int CurrentSchemaVersion = 8;
private readonly ControlOptions _options = options.Value;
private readonly string _connectionString = new SqliteConnectionStringBuilder
{
@@ -124,9 +124,174 @@ public sealed class ControlStore(IOptions options)
CREATE UNIQUE INDEX IF NOT EXISTS ux_links_active_policy
ON links(source_node_id, target_node_id, protocol, port)
WHERE desired_state = 'Active';
- PRAGMA user_version = 1;
""";
await command.ExecuteNonQueryAsync(cancellationToken);
+
+ if (schemaVersion < 1)
+ {
+ var markVersionOne = connection.CreateCommand();
+ markVersionOne.CommandText = "PRAGMA user_version = 1;";
+ await markVersionOne.ExecuteNonQueryAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 2)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var migrateProvisioning = connection.CreateCommand();
+ migrateProvisioning.Transaction = migration;
+ migrateProvisioning.CommandText = """
+ CREATE TABLE IF NOT EXISTS provisioning_jobs (
+ id TEXT PRIMARY KEY,
+ node_id TEXT NOT NULL REFERENCES agents(node_id) ON DELETE CASCADE,
+ action_type TEXT NOT NULL,
+ schema_version INTEGER NOT NULL,
+ parameters_json TEXT NOT NULL,
+ state TEXT NOT NULL,
+ confirmation_required INTEGER NOT NULL,
+ audit_reason TEXT NOT NULL,
+ created_by TEXT NOT NULL,
+ created_at TEXT NOT NULL,
+ updated_at TEXT NOT NULL,
+ expires_at TEXT NOT NULL,
+ confirmed_at TEXT NULL,
+ cancelled_at TEXT NULL,
+ version INTEGER NOT NULL,
+ last_error TEXT NULL
+ );
+ CREATE INDEX IF NOT EXISTS ix_provisioning_jobs_node_created
+ ON provisioning_jobs(node_id, created_at DESC);
+ CREATE UNIQUE INDEX IF NOT EXISTS ux_provisioning_jobs_active_node
+ ON provisioning_jobs(node_id)
+ WHERE state NOT IN ('Completed', 'Cancelled', 'Failed', 'RolledBack', 'RollbackFailed');
+ CREATE TABLE IF NOT EXISTS provisioning_events (
+ sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL REFERENCES provisioning_jobs(id) ON DELETE CASCADE,
+ recorded_at TEXT NOT NULL,
+ event_type TEXT NOT NULL,
+ state TEXT NOT NULL,
+ message TEXT NOT NULL
+ );
+ CREATE INDEX IF NOT EXISTS ix_provisioning_events_job_sequence
+ ON provisioning_events(job_id, sequence);
+ PRAGMA user_version = 2;
+ """;
+ await migrateProvisioning.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 3)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var migrateProgress = connection.CreateCommand();
+ migrateProgress.Transaction = migration;
+ migrateProgress.CommandText = """
+ ALTER TABLE provisioning_jobs
+ ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0;
+ ALTER TABLE provisioning_jobs
+ ADD COLUMN current_step TEXT NOT NULL DEFAULT '';
+ PRAGMA user_version = 3;
+ """;
+ await migrateProgress.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 4)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var strengthenProvisioningLock = connection.CreateCommand();
+ strengthenProvisioningLock.Transaction = migration;
+ strengthenProvisioningLock.CommandText = """
+ DROP INDEX IF EXISTS ux_provisioning_jobs_active_node;
+ CREATE UNIQUE INDEX ux_provisioning_jobs_active_node
+ ON provisioning_jobs(node_id)
+ WHERE state NOT IN ('Completed', 'Cancelled', 'RolledBack');
+ PRAGMA user_version = 4;
+ """;
+ await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 5)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var addStructuredEventFields = connection.CreateCommand();
+ addStructuredEventFields.Transaction = migration;
+ addStructuredEventFields.CommandText = """
+ ALTER TABLE provisioning_events
+ ADD COLUMN step TEXT NOT NULL DEFAULT '';
+ ALTER TABLE provisioning_events
+ ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0;
+ PRAGMA user_version = 5;
+ """;
+ await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 6)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var addNodePreflightFacts = connection.CreateCommand();
+ addNodePreflightFacts.Transaction = migration;
+ addNodePreflightFacts.CommandText = """
+ CREATE TABLE IF NOT EXISTS node_preflight_facts (
+ node_id TEXT PRIMARY KEY REFERENCES agents(node_id) ON DELETE CASCADE,
+ schema_version INTEGER NOT NULL,
+ facts_json TEXT NOT NULL,
+ observed_at TEXT NOT NULL,
+ source_job_id TEXT NOT NULL REFERENCES provisioning_jobs(id),
+ updated_at TEXT NOT NULL
+ );
+ PRAGMA user_version = 6;
+ """;
+ await addNodePreflightFacts.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 7)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var addPreflightDesiredState = connection.CreateCommand();
+ addPreflightDesiredState.Transaction = migration;
+ addPreflightDesiredState.CommandText = """
+ CREATE TABLE IF NOT EXISTS node_preflight_desired_state (
+ node_id TEXT PRIMARY KEY REFERENCES agents(node_id) ON DELETE CASCADE,
+ schema_version INTEGER NOT NULL,
+ desired_json TEXT NOT NULL,
+ version INTEGER NOT NULL,
+ updated_by TEXT NOT NULL,
+ audit_reason TEXT NOT NULL,
+ updated_at TEXT NOT NULL
+ );
+ PRAGMA user_version = 7;
+ """;
+ await addPreflightDesiredState.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
+
+ if (schemaVersion < 8)
+ {
+ await using var migration =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var addBaseInstallPlans = connection.CreateCommand();
+ addBaseInstallPlans.Transaction = migration;
+ addBaseInstallPlans.CommandText = """
+ CREATE TABLE IF NOT EXISTS provisioning_base_install_plans (
+ job_id TEXT PRIMARY KEY REFERENCES provisioning_jobs(id) ON DELETE CASCADE,
+ schema_version INTEGER NOT NULL,
+ plan_json TEXT NOT NULL,
+ created_at TEXT NOT NULL
+ );
+ PRAGMA user_version = 8;
+ """;
+ await addBaseInstallPlans.ExecuteNonQueryAsync(cancellationToken);
+ await migration.CommitAsync(cancellationToken);
+ }
}
public async Task MaintainAsync(
@@ -134,8 +299,53 @@ public sealed class ControlStore(IOptions options)
CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
var command = connection.CreateCommand();
+ command.Transaction = transaction;
command.CommandText = """
+ INSERT INTO provisioning_events(
+ job_id, recorded_at, event_type, state, message, step, progress_percent)
+ SELECT id, $now, 'job.expired', 'Cancelled',
+ 'Provisioning job expired before execution.', 'expired', progress_percent
+ FROM provisioning_jobs
+ WHERE expires_at <= $now
+ AND state IN ('Queued', 'AwaitingConfirmation');
+ UPDATE provisioning_jobs SET
+ state = 'Cancelled', cancelled_at = $now, updated_at = $now,
+ current_step = 'expired', version = version + 1,
+ last_error = 'job.ttl_expired'
+ WHERE expires_at <= $now
+ AND state IN ('Queued', 'AwaitingConfirmation');
+ SELECT changes();
+ INSERT INTO provisioning_events(
+ job_id, recorded_at, event_type, state, message, step, progress_percent)
+ SELECT id, $now, 'job.reconciliation.required', 'NeedsReconciliation',
+ 'Execution lease expired; factual state must be inspected.',
+ 'reconcile', progress_percent
+ FROM provisioning_jobs
+ WHERE expires_at <= $now
+ AND state IN ('Preflight', 'Running', 'Verifying');
+ UPDATE provisioning_jobs SET
+ state = 'NeedsReconciliation', updated_at = $now,
+ current_step = 'reconcile', version = version + 1,
+ last_error = 'job.ttl_expired'
+ WHERE expires_at <= $now
+ AND state IN ('Preflight', 'Running', 'Verifying');
+ SELECT changes();
+ INSERT INTO provisioning_events(
+ job_id, recorded_at, event_type, state, message, step, progress_percent)
+ SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation',
+ 'Rollback lease expired; factual state must be inspected.',
+ 'rollback-reconcile', progress_percent
+ FROM provisioning_jobs
+ WHERE expires_at <= $now AND state = 'RollingBack';
+ UPDATE provisioning_jobs SET
+ state = 'NeedsReconciliation', updated_at = $now,
+ current_step = 'rollback-reconcile', version = version + 1,
+ last_error = 'job.rollback.ttl_expired'
+ WHERE expires_at <= $now AND state = 'RollingBack';
+ SELECT changes();
DELETE FROM metric_samples WHERE recorded_at < $metric_cutoff;
SELECT changes();
DELETE FROM idempotency WHERE created_at < $idempotency_cutoff;
@@ -156,24 +366,26 @@ public sealed class ControlStore(IOptions options)
command.Parameters.AddWithValue(
"$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O"));
command.Parameters.AddWithValue("$now", now.ToString("O"));
- var deleted = new int[6];
+ var changes = new int[9];
await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
{
- for (var index = 0; index < deleted.Length; index++)
+ for (var index = 0; index < changes.Length; index++)
{
if (await reader.ReadAsync(cancellationToken))
{
- deleted[index] = reader.GetInt32(0);
+ changes[index] = reader.GetInt32(0);
}
await reader.NextResultAsync(cancellationToken);
}
}
+ await transaction.CommitAsync(cancellationToken);
var optimize = connection.CreateCommand();
optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);";
await optimize.ExecuteNonQueryAsync(cancellationToken);
return new ControlMaintenanceResult(
- deleted[0], deleted[1], deleted[2], deleted[3] + deleted[4] + deleted[5]);
+ changes[3], changes[4], changes[5], changes[6] + changes[7] + changes[8],
+ changes[0], changes[1] + changes[2]);
}
public async Task BackupDatabaseAsync(string destinationPath, CancellationToken cancellationToken = default)
diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs
index 21a3bc8..12a3206 100644
--- a/src/ServerMonitorManager.Control/Program.cs
+++ b/src/ServerMonitorManager.Control/Program.cs
@@ -377,10 +377,398 @@ agents.MapPost("/heartbeat", async (
});
}
});
+agents.MapGet("/provisioning/jobs/next", async (
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
+ if (string.IsNullOrWhiteSpace(nodeId) || !NodeIdValidator.IsValid(nodeId))
+ {
+ return Results.Forbid();
+ }
+
+ var job = await controlStore.ClaimNextProvisioningJobAsync(nodeId, cancellationToken);
+ return job is null ? Results.NoContent() : Results.Ok(job);
+});
+agents.MapPost("/provisioning/jobs/{id}/progress", async (
+ string id,
+ ProvisioningJobProgressRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
+ if (string.IsNullOrWhiteSpace(nodeId)
+ || !NodeIdValidator.IsValid(nodeId)
+ || !ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningProgress"] =
+ ["Invalid job id, state, progress, step, event code, message, or idempotency key."]
+ });
+ }
+
+ try
+ {
+ var job = await controlStore.ReportProvisioningProgressAsync(
+ nodeId, id, request, cancellationToken);
+ return job is null ? Results.NotFound() : Results.Ok(job);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+});
+agents.MapPost("/provisioning/jobs/{id}/preflight-facts", async (
+ string id,
+ ProvisioningPreflightReportRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
+ if (string.IsNullOrWhiteSpace(nodeId)
+ || !NodeIdValidator.IsValid(nodeId)
+ || !ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["preflightFacts"] = ["Invalid job id, facts, observation time, or idempotency key."]
+ });
+ }
+
+ try
+ {
+ var facts = await controlStore.RecordPreflightFactsAsync(
+ nodeId, id, request, cancellationToken);
+ return facts is null ? Results.NotFound() : Results.Ok(facts);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+});
+agents.MapPost("/provisioning/jobs/{id}/base-install-plan", async (
+ string id,
+ SystemBaseInstallPlanReportRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
+ if (string.IsNullOrWhiteSpace(nodeId)
+ || !NodeIdValidator.IsValid(nodeId)
+ || !ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["baseInstallPlan"] = ["Invalid job id, plan, or idempotency key."]
+ });
+ }
+
+ try
+ {
+ var plan = await controlStore.RecordBaseInstallPlanAsync(
+ nodeId, id, request, cancellationToken);
+ return plan is null ? Results.NotFound() : Results.Ok(plan);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+ catch (ProvisioningPlanValidationException exception)
+ {
+ return Results.BadRequest(new ProblemDetails { Title = exception.Message });
+ }
+});
+agents.MapPost("/provisioning/jobs/{id}/execution-grant", async (
+ string id,
+ ProvisioningExecutionGrantRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CertificateAuthority authority,
+ TimeProvider timeProvider,
+ CancellationToken cancellationToken) =>
+{
+ var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
+ if (string.IsNullOrWhiteSpace(nodeId)
+ || !NodeIdValidator.IsValid(nodeId)
+ || !ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["executionGrant"] = ["Invalid job id or idempotency key."]
+ });
+ }
+ try
+ {
+ var grant = await controlStore.IssueBaseInstallExecutionGrantAsync(
+ nodeId,
+ id,
+ request,
+ (job, plan) => authority.SignProvisioningExecutionGrant(
+ job, plan, timeProvider.GetUtcNow(), TimeSpan.FromMinutes(2)),
+ cancellationToken);
+ return grant is null ? Results.NotFound() : Results.Ok(grant);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+});
var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator");
control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) =>
Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray()));
+control.MapGet("/provisioning/catalogs/system-base-install/1", () =>
+ Results.Ok(SystemBaseInstallCatalogDefinition.Create()));
+control.MapGet("/agents/{nodeId}/facts/preflight", async (
+ string nodeId,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!NodeIdValidator.IsValid(nodeId))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["nodeId"] = ["Invalid node id."]
+ });
+ }
+ var facts = await controlStore.GetPreflightFactsAsync(nodeId, cancellationToken);
+ return facts is null ? Results.NotFound() : Results.Ok(facts);
+});
+control.MapPut("/agents/{nodeId}/desired/preflight", async (
+ string nodeId,
+ PreflightDesiredStateUpdateRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!NodeIdValidator.IsValid(nodeId) || !PreflightDesiredStateValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["preflightDesiredState"] =
+ ["Invalid schema, requirements, architectures, audit reason, or idempotency key."]
+ });
+ }
+ try
+ {
+ var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+ var desired = await controlStore.SetPreflightDesiredStateAsync(
+ nodeId, request, actor, cancellationToken);
+ return Results.Ok(desired);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningNodeNotFoundException)
+ {
+ return Results.NotFound();
+ }
+});
+control.MapGet("/agents/{nodeId}/drift/preflight", async (
+ string nodeId,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!NodeIdValidator.IsValid(nodeId))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["nodeId"] = ["Invalid node id."]
+ });
+ }
+ var assessment = await controlStore.AssessPreflightDriftAsync(nodeId, cancellationToken);
+ return assessment is null ? Results.NotFound() : Results.Ok(assessment);
+});
+control.MapPost("/agents/{nodeId}/provisioning/jobs", async (
+ string nodeId,
+ ProvisioningJobCreateRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!NodeIdValidator.IsValid(nodeId) || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningJob"] =
+ ["Invalid node id, action schema, parameters, TTL, audit reason, or idempotency key."]
+ });
+ }
+
+ try
+ {
+ var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+ var job = await controlStore.CreateProvisioningJobAsync(
+ nodeId, request, actor, cancellationToken);
+ return Results.Created($"/api/v1/control/provisioning/jobs/{job.Id}", job);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningNodeNotFoundException)
+ {
+ return Results.NotFound();
+ }
+ catch (SqliteException exception) when (exception.SqliteErrorCode == 19)
+ {
+ return Results.Conflict(new ProblemDetails
+ {
+ Title = "The node already has an incompatible active provisioning job."
+ });
+ }
+});
+control.MapGet("/provisioning/jobs/{id}", async (
+ string id,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!ProvisioningJobValidator.IsValidId(id))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningJob"] = ["Invalid provisioning job id."]
+ });
+ }
+ var job = await controlStore.GetProvisioningJobAsync(id, cancellationToken);
+ return job is null ? Results.NotFound() : Results.Ok(job);
+});
+control.MapGet("/provisioning/jobs/{id}/plan", async (
+ string id,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!ProvisioningJobValidator.IsValidId(id))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["baseInstallPlan"] = ["Invalid provisioning job id."]
+ });
+ }
+ var plan = await controlStore.GetBaseInstallPlanAsync(id, cancellationToken);
+ return plan is null ? Results.NotFound() : Results.Ok(plan);
+});
+control.MapGet("/provisioning/jobs/{id}/events", async (
+ string id,
+ int? limit,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!ProvisioningJobValidator.IsValidId(id) || limit is < 1 or > 200)
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningEvents"] = ["Invalid provisioning job id or limit (1-200)."]
+ });
+ }
+ var events = await controlStore.ListProvisioningEventsAsync(
+ id, limit ?? 100, cancellationToken);
+ return events is null ? Results.NotFound() : Results.Ok(events.ToArray());
+});
+control.MapPost("/provisioning/jobs/{id}/confirm", async (
+ string id,
+ ProvisioningJobCommandRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+ await ChangeProvisioningJobAsync(
+ id, request, context, controlStore, confirm: true, cancellationToken));
+control.MapPost("/provisioning/jobs/{id}/cancel", async (
+ string id,
+ ProvisioningJobCommandRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+ await ChangeProvisioningJobAsync(
+ id, request, context, controlStore, confirm: false, cancellationToken));
+control.MapPost("/provisioning/jobs/{id}/retry", async (
+ string id,
+ ProvisioningJobCommandRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."]
+ });
+ }
+ try
+ {
+ var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+ var job = await controlStore.RetryProvisioningJobAsync(
+ id, request, actor, cancellationToken);
+ return job is null ? Results.NotFound() : Results.Ok(job);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+});
+control.MapPost("/provisioning/jobs/{id}/rollback", async (
+ string id,
+ ProvisioningJobCommandRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ CancellationToken cancellationToken) =>
+{
+ if (!ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."]
+ });
+ }
+ try
+ {
+ var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+ var job = await controlStore.StartProvisioningRollbackAsync(
+ id, request, actor, cancellationToken);
+ return job is null ? Results.NotFound() : Results.Ok(job);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+});
control.MapPost("/automations/token", async (
AutomationTokenCreateRequest request,
HttpContext context,
@@ -577,6 +965,41 @@ automation.MapGet("/links", async (
await app.RunAsync();
return 0;
+static async Task ChangeProvisioningJobAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ HttpContext context,
+ ControlStore controlStore,
+ bool confirm,
+ CancellationToken cancellationToken)
+{
+ if (!ProvisioningJobValidator.IsValidId(id)
+ || !ProvisioningJobValidator.IsValid(request))
+ {
+ return Results.ValidationProblem(new Dictionary
+ {
+ ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."]
+ });
+ }
+
+ try
+ {
+ var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+ var job = confirm
+ ? await controlStore.ConfirmProvisioningJobAsync(id, request, actor, cancellationToken)
+ : await controlStore.CancelProvisioningJobAsync(id, request, actor, cancellationToken);
+ return job is null ? Results.NotFound() : Results.Ok(job);
+ }
+ catch (IdempotencyConflictException)
+ {
+ return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+ }
+ catch (ProvisioningTransitionException exception)
+ {
+ return Results.Conflict(new ProblemDetails { Title = exception.Message });
+ }
+}
+
public partial class Program;
internal static class NodeIdValidator
@@ -611,3 +1034,103 @@ internal static class CertificateReenrollmentValidator
=> request.Reason.Length is >= 1 and <= 200
&& IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
}
+
+internal static class ProvisioningJobValidator
+{
+ private const int MaximumParametersBytes = 16 * 1024;
+
+ public static bool IsValid(ProvisioningJobCreateRequest request)
+ {
+ if (request.SchemaVersion != 1
+ || request.Parameters.ValueKind != JsonValueKind.Object
+ || request.Parameters.GetRawText().Length > MaximumParametersBytes
+ || request.TtlMinutes is < 5 or > 1440
+ || request.AuditReason is not { Length: >= 1 and <= 256 }
+ || !IdempotencyKeyValidator.IsValid(request.IdempotencyKey))
+ {
+ return false;
+ }
+ if (request.ActionType == "preflight")
+ {
+ return !request.Parameters.EnumerateObject().Any();
+ }
+ if (request.ActionType != "system.base-install")
+ {
+ return false;
+ }
+ return SystemBaseInstallSchema.TryParse(request.Parameters, out _);
+ }
+
+ public static bool IsValid(ProvisioningJobCommandRequest request)
+ => request.Reason.Length is >= 1 and <= 256
+ && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+
+ public static bool IsValid(ProvisioningJobProgressRequest request)
+ => request.State is ProvisioningJobStates.Preflight
+ or ProvisioningJobStates.Running
+ or ProvisioningJobStates.Verifying
+ or ProvisioningJobStates.Completed
+ or ProvisioningJobStates.Failed
+ or ProvisioningJobStates.NeedsReconciliation
+ or ProvisioningJobStates.RollingBack
+ or ProvisioningJobStates.RolledBack
+ or ProvisioningJobStates.RollbackFailed
+ && request.ProgressPercent is >= 0 and <= 100
+ && IsSafeCode(request.Step, 64)
+ && IsSafeCode(request.EventCode, 64)
+ && request.Message.Length <= 512
+ && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+
+ public static bool IsValid(ProvisioningPreflightReportRequest request)
+ => request.ObservedAt >= DateTimeOffset.UtcNow.AddHours(-1)
+ && request.ObservedAt <= DateTimeOffset.UtcNow.AddMinutes(1)
+ && request.Facts is not null
+ && IsSafeFact(request.Facts.OperatingSystem, 32)
+ && IsSafeFact(request.Facts.OperatingSystemVersion, 64)
+ && request.Facts.Architecture is "x64" or "x86" or "arm" or "arm64"
+ && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+
+ public static bool IsValid(SystemBaseInstallPlanReportRequest request)
+ => request.Plan is not null
+ && request.Plan.Packages is { Length: <= 64 }
+ && request.Plan.Warnings is { Length: <= 2 }
+ && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+
+ public static bool IsValid(ProvisioningExecutionGrantRequest request)
+ => IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+
+ public static bool IsValidId(string id)
+ => id.Length == 32 && Guid.TryParseExact(id, "N", out _);
+
+ public static bool RequiresConfirmation(string actionType)
+ => actionType == "system.base-install";
+
+ private static bool IsSafeCode(string value, int maximumLength)
+ => value.Length is >= 1 && value.Length <= maximumLength
+ && value.All(character => character is >= 'a' and <= 'z'
+ or >= '0' and <= '9'
+ or '.' or '-' or '_');
+
+ private static bool IsSafeFact(string? value, int maximumLength)
+ => value is not null
+ && value.Length is >= 1 && value.Length <= maximumLength
+ && value.All(character => char.IsAsciiLetterOrDigit(character)
+ || character is '_' or '-' or '.');
+
+}
+
+internal static class PreflightDesiredStateValidator
+{
+ private static readonly HashSet SupportedArchitectures =
+ new(["x64", "x86", "arm", "arm64"], StringComparer.Ordinal);
+
+ public static bool IsValid(PreflightDesiredStateUpdateRequest request)
+ => request.SchemaVersion == 1
+ && request.Desired is not null
+ && request.Desired.AllowedArchitectures is { Length: >= 1 and <= 4 }
+ && request.Desired.AllowedArchitectures.Distinct(StringComparer.Ordinal).Count()
+ == request.Desired.AllowedArchitectures.Length
+ && request.Desired.AllowedArchitectures.All(SupportedArchitectures.Contains)
+ && request.AuditReason is { Length: >= 1 and <= 256 }
+ && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
+}
diff --git a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs
new file mode 100644
index 0000000..8abbe0b
--- /dev/null
+++ b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs
@@ -0,0 +1,209 @@
+using System.Text.Json;
+using Microsoft.Data.Sqlite;
+using ServerMonitorManager.Core;
+
+namespace ServerMonitorManager.Control;
+
+public sealed partial class ControlStore
+{
+ public async Task RecordBaseInstallPlanAsync(
+ string nodeId,
+ string jobId,
+ SystemBaseInstallPlanReportRequest request,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-base-install-plan:{nodeId}:{jobId}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(
+ request, SmmJsonContext.Default.SystemBaseInstallPlanReportRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningBaseInstallPlanRecord, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken);
+ if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal))
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (job.ActionType != "system.base-install" || job.SchemaVersion != 1
+ || job.State != ProvisioningJobStates.Preflight || !job.ConfirmationRequired
+ || job.ConfirmedAt is not null)
+ {
+ throw new ProvisioningTransitionException(job.State, ProvisioningJobStates.AwaitingConfirmation);
+ }
+ if (!SystemBaseInstallSchema.TryParse(job.Parameters, out var parameters)
+ || parameters is null
+ || !SystemBaseInstallSchema.IsValidPlan(parameters, request.Plan))
+ {
+ throw new ProvisioningPlanValidationException();
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var record = new ProvisioningBaseInstallPlanRecord(
+ jobId, nodeId, job.SchemaVersion, request.Plan, now);
+ var insert = connection.CreateCommand();
+ insert.Transaction = transaction;
+ insert.CommandText = """
+ INSERT INTO provisioning_base_install_plans(
+ job_id, schema_version, plan_json, created_at)
+ VALUES ($job, $schema, $plan, $created);
+ """;
+ insert.Parameters.AddWithValue("$job", record.JobId);
+ insert.Parameters.AddWithValue("$schema", record.SchemaVersion);
+ insert.Parameters.AddWithValue(
+ "$plan", JsonSerializer.Serialize(record.Plan, SmmJsonContext.Default.SystemBaseInstallPlan));
+ insert.Parameters.AddWithValue("$created", record.CreatedAt.ToString("O"));
+ await insert.ExecuteNonQueryAsync(cancellationToken);
+
+ var update = connection.CreateCommand();
+ update.Transaction = transaction;
+ update.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = $state,
+ progress_percent = 25,
+ current_step = 'awaiting-confirmation',
+ updated_at = $updated,
+ version = version + 1
+ WHERE id = $id AND node_id = $node AND version = $version
+ AND state = $preflight AND confirmed_at IS NULL;
+ """;
+ update.Parameters.AddWithValue("$state", ProvisioningJobStates.AwaitingConfirmation);
+ update.Parameters.AddWithValue("$updated", now.ToString("O"));
+ update.Parameters.AddWithValue("$id", job.Id);
+ update.Parameters.AddWithValue("$node", nodeId);
+ update.Parameters.AddWithValue("$version", job.Version);
+ update.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight);
+ if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+ {
+ throw new ProvisioningTransitionException(job.State, ProvisioningJobStates.AwaitingConfirmation);
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, jobId, "base-install.plan-recorded",
+ ProvisioningJobStates.AwaitingConfirmation,
+ "Validated base installation plan is awaiting operator confirmation.", now,
+ cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, record,
+ SmmJsonContext.Default.ProvisioningBaseInstallPlanRecord, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, nodeId, "provisioning.base-install.plan", jobId,
+ JsonSerializer.Serialize(new
+ {
+ record.SchemaVersion,
+ PackageCount = record.Plan.Packages.Length,
+ WarningCodes = record.Plan.Warnings
+ }),
+ cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return record;
+ }
+
+ public async Task GetBaseInstallPlanAsync(
+ string jobId,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ var command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT p.schema_version, p.plan_json, p.created_at, j.node_id
+ FROM provisioning_base_install_plans p
+ INNER JOIN provisioning_jobs j ON j.id = p.job_id
+ WHERE p.job_id = $job;
+ """;
+ command.Parameters.AddWithValue("$job", jobId);
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ if (!await reader.ReadAsync(cancellationToken))
+ {
+ return null;
+ }
+ var plan = JsonSerializer.Deserialize(
+ reader.GetString(1), SmmJsonContext.Default.SystemBaseInstallPlan)
+ ?? throw new InvalidDataException("Stored base installation plan is invalid.");
+ return new ProvisioningBaseInstallPlanRecord(
+ jobId, reader.GetString(3), reader.GetInt32(0), plan,
+ DateTimeOffset.Parse(reader.GetString(2)));
+ }
+
+ public async Task IssueBaseInstallExecutionGrantAsync(
+ string nodeId,
+ string jobId,
+ ProvisioningExecutionGrantRequest request,
+ Func issueGrant,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-execution-grant:{nodeId}:{jobId}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(
+ request, SmmJsonContext.Default.ProvisioningExecutionGrantRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken);
+ if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal))
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (job.ActionType != "system.base-install"
+ || job.SchemaVersion != 1
+ || job.State != ProvisioningJobStates.Queued
+ || !job.ConfirmationRequired
+ || job.ConfirmedAt is null
+ || job.ExpiresAt <= DateTimeOffset.UtcNow)
+ {
+ throw new ProvisioningTransitionException(job.State, "ExecutionGrant");
+ }
+
+ var planCommand = connection.CreateCommand();
+ planCommand.Transaction = transaction;
+ planCommand.CommandText = """
+ SELECT plan_json FROM provisioning_base_install_plans
+ WHERE job_id = $job AND schema_version = 1;
+ """;
+ planCommand.Parameters.AddWithValue("$job", jobId);
+ var planJson = await planCommand.ExecuteScalarAsync(cancellationToken) as string;
+ if (planJson is null)
+ {
+ throw new ProvisioningTransitionException(job.State, "ExecutionGrant");
+ }
+ var plan = JsonSerializer.Deserialize(
+ planJson, SmmJsonContext.Default.SystemBaseInstallPlan)
+ ?? throw new InvalidDataException("Stored base installation plan is invalid.");
+ var grant = issueGrant(job, plan);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, grant,
+ SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, nodeId, "provisioning.execution-grant.issued", jobId,
+ JsonSerializer.Serialize(new
+ {
+ grant.ActionType,
+ grant.SchemaVersion,
+ grant.PlanSha256,
+ grant.ExpiresAtUnixSeconds
+ }),
+ cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return grant;
+ }
+}
+
+public sealed class ProvisioningPlanValidationException()
+ : Exception("Base installation plan does not match the validated job parameters.");
diff --git a/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs
new file mode 100644
index 0000000..94d02b4
--- /dev/null
+++ b/src/ServerMonitorManager.Control/ProvisioningFactsStore.cs
@@ -0,0 +1,272 @@
+using System.Text.Json;
+using Microsoft.Data.Sqlite;
+using ServerMonitorManager.Core;
+
+namespace ServerMonitorManager.Control;
+
+public sealed partial class ControlStore
+{
+ public async Task RecordPreflightFactsAsync(
+ string nodeId,
+ string jobId,
+ ProvisioningPreflightReportRequest request,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-preflight:{nodeId}:{jobId}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningPreflightReportRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.NodePreflightFacts, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken);
+ if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal))
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (job.ActionType != "preflight" || job.SchemaVersion != 1
+ || job.State != ProvisioningJobStates.Preflight)
+ {
+ throw new ProvisioningTransitionException(job.State, "PreflightFacts");
+ }
+
+ var facts = new NodePreflightFacts(
+ nodeId, 1, request.Facts, request.ObservedAt, jobId, DateTimeOffset.UtcNow);
+ var upsert = connection.CreateCommand();
+ upsert.Transaction = transaction;
+ upsert.CommandText = """
+ INSERT INTO node_preflight_facts(
+ node_id, schema_version, facts_json, observed_at, source_job_id, updated_at)
+ VALUES ($node, $schema, $facts, $observed, $job, $updated)
+ ON CONFLICT(node_id) DO UPDATE SET
+ schema_version = excluded.schema_version,
+ facts_json = excluded.facts_json,
+ observed_at = excluded.observed_at,
+ source_job_id = excluded.source_job_id,
+ updated_at = excluded.updated_at;
+ """;
+ upsert.Parameters.AddWithValue("$node", facts.NodeId);
+ upsert.Parameters.AddWithValue("$schema", facts.SchemaVersion);
+ upsert.Parameters.AddWithValue(
+ "$facts", JsonSerializer.Serialize(facts.Facts, SmmJsonContext.Default.ProvisioningPreflightResult));
+ upsert.Parameters.AddWithValue("$observed", facts.ObservedAt.ToString("O"));
+ upsert.Parameters.AddWithValue("$job", facts.SourceJobId);
+ upsert.Parameters.AddWithValue("$updated", facts.UpdatedAt.ToString("O"));
+ await upsert.ExecuteNonQueryAsync(cancellationToken);
+ await WriteProvisioningEventAsync(
+ connection, transaction, jobId, "preflight.facts-recorded", job.State,
+ "Validated preflight facts were recorded.", facts.UpdatedAt, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, facts,
+ SmmJsonContext.Default.NodePreflightFacts, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, nodeId, "provisioning.preflight.facts", jobId,
+ JsonSerializer.Serialize(new { facts.SchemaVersion, facts.ObservedAt }), cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return facts;
+ }
+
+ public async Task GetPreflightFactsAsync(
+ string nodeId,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ var command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT schema_version, facts_json, observed_at, source_job_id, updated_at
+ FROM node_preflight_facts WHERE node_id = $node;
+ """;
+ command.Parameters.AddWithValue("$node", nodeId);
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ if (!await reader.ReadAsync(cancellationToken))
+ {
+ return null;
+ }
+ var facts = JsonSerializer.Deserialize(
+ reader.GetString(1), SmmJsonContext.Default.ProvisioningPreflightResult)
+ ?? throw new InvalidDataException("Stored preflight facts are invalid.");
+ return new NodePreflightFacts(
+ nodeId, reader.GetInt32(0), facts, DateTimeOffset.Parse(reader.GetString(2)),
+ reader.GetString(3), DateTimeOffset.Parse(reader.GetString(4)));
+ }
+
+ public async Task SetPreflightDesiredStateAsync(
+ string nodeId,
+ PreflightDesiredStateUpdateRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction =
+ (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"preflight-desired:{actor}:{nodeId}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.PreflightDesiredStateUpdateRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.NodePreflightDesiredState, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var exists = connection.CreateCommand();
+ exists.Transaction = transaction;
+ exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);";
+ exists.Parameters.AddWithValue("$node", nodeId);
+ if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1)
+ {
+ throw new ProvisioningNodeNotFoundException(nodeId);
+ }
+
+ var readVersion = connection.CreateCommand();
+ readVersion.Transaction = transaction;
+ readVersion.CommandText =
+ "SELECT version FROM node_preflight_desired_state WHERE node_id = $node;";
+ readVersion.Parameters.AddWithValue("$node", nodeId);
+ var existingVersion = await readVersion.ExecuteScalarAsync(cancellationToken);
+ var now = DateTimeOffset.UtcNow;
+ var desired = new NodePreflightDesiredState(
+ nodeId, request.SchemaVersion, request.Desired,
+ existingVersion is null ? 1 : Convert.ToInt64(existingVersion) + 1,
+ actor, request.AuditReason, now);
+ var upsert = connection.CreateCommand();
+ upsert.Transaction = transaction;
+ upsert.CommandText = """
+ INSERT INTO node_preflight_desired_state(
+ node_id, schema_version, desired_json, version,
+ updated_by, audit_reason, updated_at)
+ VALUES ($node, $schema, $desired, $version, $actor, $reason, $updated)
+ ON CONFLICT(node_id) DO UPDATE SET
+ schema_version = excluded.schema_version,
+ desired_json = excluded.desired_json,
+ version = excluded.version,
+ updated_by = excluded.updated_by,
+ audit_reason = excluded.audit_reason,
+ updated_at = excluded.updated_at;
+ """;
+ upsert.Parameters.AddWithValue("$node", desired.NodeId);
+ upsert.Parameters.AddWithValue("$schema", desired.SchemaVersion);
+ upsert.Parameters.AddWithValue(
+ "$desired",
+ JsonSerializer.Serialize(desired.Desired, SmmJsonContext.Default.PreflightDesiredRequirements));
+ upsert.Parameters.AddWithValue("$version", desired.Version);
+ upsert.Parameters.AddWithValue("$actor", desired.UpdatedBy);
+ upsert.Parameters.AddWithValue("$reason", desired.AuditReason);
+ upsert.Parameters.AddWithValue("$updated", desired.UpdatedAt.ToString("O"));
+ await upsert.ExecuteNonQueryAsync(cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, desired,
+ SmmJsonContext.Default.NodePreflightDesiredState, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, actor, "provisioning.preflight.desired", nodeId,
+ JsonSerializer.Serialize(new { desired.SchemaVersion, desired.Version }), cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return desired;
+ }
+
+ public async Task GetPreflightDesiredStateAsync(
+ string nodeId,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ var command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT schema_version, desired_json, version, updated_by, audit_reason, updated_at
+ FROM node_preflight_desired_state WHERE node_id = $node;
+ """;
+ command.Parameters.AddWithValue("$node", nodeId);
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ if (!await reader.ReadAsync(cancellationToken))
+ {
+ return null;
+ }
+ var desired = JsonSerializer.Deserialize(
+ reader.GetString(1), SmmJsonContext.Default.PreflightDesiredRequirements)
+ ?? throw new InvalidDataException("Stored preflight desired state is invalid.");
+ return new NodePreflightDesiredState(
+ nodeId, reader.GetInt32(0), desired, reader.GetInt64(2), reader.GetString(3),
+ reader.GetString(4), DateTimeOffset.Parse(reader.GetString(5)));
+ }
+
+ public async Task AssessPreflightDriftAsync(
+ string nodeId,
+ CancellationToken cancellationToken = default)
+ {
+ await using (var connection = await OpenAsync(cancellationToken))
+ {
+ var exists = connection.CreateCommand();
+ exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);";
+ exists.Parameters.AddWithValue("$node", nodeId);
+ if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1)
+ {
+ return null;
+ }
+ }
+ var desired = await GetPreflightDesiredStateAsync(nodeId, cancellationToken);
+ var facts = await GetPreflightFactsAsync(nodeId, cancellationToken);
+ return PreflightDriftEvaluator.Assess(nodeId, desired, facts);
+ }
+}
+
+internal static class PreflightDriftEvaluator
+{
+ public static PreflightDriftAssessment Assess(
+ string nodeId,
+ NodePreflightDesiredState? desired,
+ NodePreflightFacts? facts)
+ {
+ if (desired is null)
+ {
+ return new PreflightDriftAssessment(
+ nodeId, PreflightDriftStatuses.NotConfigured, [], null, facts);
+ }
+ if (facts is null)
+ {
+ return new PreflightDriftAssessment(
+ nodeId, PreflightDriftStatuses.Unknown, [PreflightDriftCodes.FactsMissing], desired, null);
+ }
+
+ var drift = new List();
+ AddMissing(
+ drift, desired.Desired.RequireSystemd, facts.Facts.HasSystemd,
+ PreflightDriftCodes.SystemdMissing);
+ AddMissing(
+ drift, desired.Desired.RequireSshd, facts.Facts.HasSshd,
+ PreflightDriftCodes.SshdMissing);
+ AddMissing(
+ drift, desired.Desired.RequireNftables, facts.Facts.HasNftables,
+ PreflightDriftCodes.NftablesMissing);
+ AddMissing(
+ drift, desired.Desired.RequireWireGuard, facts.Facts.HasWireGuard,
+ PreflightDriftCodes.WireGuardMissing);
+ AddMissing(
+ drift, desired.Desired.RequireApt, facts.Facts.HasApt,
+ PreflightDriftCodes.AptMissing);
+ if (!desired.Desired.AllowedArchitectures.Contains(
+ facts.Facts.Architecture, StringComparer.Ordinal))
+ {
+ drift.Add(PreflightDriftCodes.ArchitectureUnsupported);
+ }
+ return new PreflightDriftAssessment(
+ nodeId,
+ drift.Count == 0 ? PreflightDriftStatuses.InSync : PreflightDriftStatuses.Drifted,
+ [.. drift], desired, facts);
+ }
+
+ private static void AddMissing(List drift, bool required, bool present, string code)
+ {
+ if (required && !present)
+ {
+ drift.Add(code);
+ }
+ }
+}
diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs
new file mode 100644
index 0000000..8cb5b30
--- /dev/null
+++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs
@@ -0,0 +1,666 @@
+using System.Text.Json;
+using Microsoft.Data.Sqlite;
+using ServerMonitorManager.Core;
+
+namespace ServerMonitorManager.Control;
+
+public sealed partial class ControlStore
+{
+ public async Task CreateProvisioningJobAsync(
+ string nodeId,
+ ProvisioningJobCreateRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-create:{actor}:{nodeId}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCreateRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var exists = connection.CreateCommand();
+ exists.Transaction = transaction;
+ exists.CommandText = "SELECT EXISTS(SELECT 1 FROM agents WHERE node_id = $node);";
+ exists.Parameters.AddWithValue("$node", nodeId);
+ if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1)
+ {
+ throw new ProvisioningNodeNotFoundException(nodeId);
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var confirmationRequired = ProvisioningJobValidator.RequiresConfirmation(request.ActionType);
+ var job = new ProvisioningJob(
+ Guid.NewGuid().ToString("N"),
+ nodeId,
+ request.ActionType,
+ request.SchemaVersion,
+ request.Parameters.Clone(),
+ ProvisioningJobStates.Queued,
+ confirmationRequired,
+ request.AuditReason,
+ actor,
+ now,
+ now,
+ now.AddMinutes(request.TtlMinutes),
+ null,
+ null,
+ 1,
+ 0,
+ "queued",
+ null);
+
+ var insert = connection.CreateCommand();
+ insert.Transaction = transaction;
+ insert.CommandText = """
+ INSERT INTO provisioning_jobs(
+ id, node_id, action_type, schema_version, parameters_json, state,
+ confirmation_required, audit_reason, created_by, created_at, updated_at,
+ expires_at, confirmed_at, cancelled_at, version, last_error)
+ VALUES(
+ $id, $node, $action, $schema, $parameters, $state,
+ $confirmation, $reason, $actor, $created, $updated,
+ $expires, NULL, NULL, $version, NULL);
+ """;
+ AddProvisioningJobParameters(insert, job);
+ await insert.ExecuteNonQueryAsync(cancellationToken);
+ await WriteProvisioningEventAsync(
+ connection, transaction, job.Id, "job.created", job.State,
+ "Provisioning job accepted by the control plane.", now, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, job,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, actor, "provisioning.job.created", job.Id,
+ JsonSerializer.Serialize(new { job.NodeId, job.ActionType, job.SchemaVersion, job.AuditReason }),
+ cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return job;
+ }
+
+ public async Task GetProvisioningJobAsync(
+ string id,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ var command = connection.CreateCommand();
+ command.CommandText = "SELECT * FROM provisioning_jobs WHERE id = $id;";
+ command.Parameters.AddWithValue("$id", id);
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ return await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null;
+ }
+
+ public async Task ClaimNextProvisioningJobAsync(
+ string nodeId,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var now = DateTimeOffset.UtcNow;
+ var command = connection.CreateCommand();
+ command.Transaction = transaction;
+ command.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END,
+ current_step = CASE
+ WHEN state = $queued THEN 'preflight'
+ ELSE 'rollback'
+ END,
+ updated_at = $now,
+ version = version + 1
+ WHERE id = (
+ SELECT id FROM provisioning_jobs
+ WHERE node_id = $node
+ AND ((state = $queued
+ AND (confirmation_required = 0 OR confirmed_at IS NULL))
+ OR (state = $rolling_back AND current_step = 'rollback-queued'))
+ AND expires_at > $now
+ ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id
+ LIMIT 1)
+ AND ((state = $queued
+ AND (confirmation_required = 0 OR confirmed_at IS NULL))
+ OR (state = $rolling_back AND current_step = 'rollback-queued'))
+ RETURNING *;
+ """;
+ command.Parameters.AddWithValue("$node", nodeId);
+ command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued);
+ command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight);
+ command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack);
+ command.Parameters.AddWithValue("$now", now.ToString("O"));
+ ProvisioningJob? job;
+ await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
+ {
+ job = await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null;
+ }
+ if (job is null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return null;
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, job.Id, "job.claimed", job.State,
+ "Provisioning job claimed by its assigned Node.", now, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, nodeId, "provisioning.job.claimed", job.Id,
+ JsonSerializer.Serialize(new { job.NodeId, job.ActionType, job.SchemaVersion }),
+ cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return job;
+ }
+
+ public Task ConfirmProvisioningJobAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ => TransitionProvisioningJobAsync(
+ id, request, actor, ProvisioningJobStates.AwaitingConfirmation,
+ ProvisioningJobStates.Queued, "job.confirmed", "provisioning.job.confirmed",
+ setConfirmedAt: true, cancellationToken);
+
+ public Task CancelProvisioningJobAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ => TransitionProvisioningJobAsync(
+ id, request, actor, null, ProvisioningJobStates.Cancelled,
+ "job.cancelled", "provisioning.job.cancelled",
+ setConfirmedAt: false, cancellationToken);
+
+ public async Task ReportProvisioningProgressAsync(
+ string nodeId,
+ string id,
+ ProvisioningJobProgressRequest request,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-progress:{nodeId}:{id}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobProgressRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
+ if (current is null || !string.Equals(current.NodeId, nodeId, StringComparison.Ordinal))
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (!ProvisioningStateMachine.CanReport(
+ current.State, request.State, current.ProgressPercent, request.ProgressPercent))
+ {
+ throw new ProvisioningTransitionException(current.State, request.State);
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var updated = current with
+ {
+ State = request.State,
+ ProgressPercent = request.ProgressPercent,
+ CurrentStep = request.Step,
+ UpdatedAt = now,
+ Version = current.Version + 1,
+ LastError = request.State is ProvisioningJobStates.Failed
+ or ProvisioningJobStates.NeedsReconciliation
+ or ProvisioningJobStates.RollbackFailed
+ ? request.EventCode
+ : null
+ };
+ var update = connection.CreateCommand();
+ update.Transaction = transaction;
+ update.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = $state,
+ progress_percent = $progress,
+ current_step = $step,
+ updated_at = $updated,
+ version = $version,
+ last_error = $error
+ WHERE id = $id AND node_id = $node AND version = $previous_version;
+ """;
+ update.Parameters.AddWithValue("$state", updated.State);
+ update.Parameters.AddWithValue("$progress", updated.ProgressPercent);
+ update.Parameters.AddWithValue("$step", updated.CurrentStep);
+ update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O"));
+ update.Parameters.AddWithValue("$version", updated.Version);
+ update.Parameters.AddWithValue("$error", (object?)updated.LastError ?? DBNull.Value);
+ update.Parameters.AddWithValue("$id", updated.Id);
+ update.Parameters.AddWithValue("$node", nodeId);
+ update.Parameters.AddWithValue("$previous_version", current.Version);
+ if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+ {
+ throw new ProvisioningTransitionException(current.State, request.State);
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, id, request.EventCode, request.State,
+ $"Agent reported '{request.EventCode}' for step '{request.Step}' at "
+ + $"{request.ProgressPercent}%.", now, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, updated,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, nodeId, "provisioning.job.progress", id,
+ JsonSerializer.Serialize(new
+ {
+ request.State,
+ request.ProgressPercent,
+ request.Step,
+ request.EventCode
+ }),
+ cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return updated;
+ }
+
+ public async Task?> ListProvisioningEventsAsync(
+ string id,
+ int limit,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ var exists = connection.CreateCommand();
+ exists.CommandText = "SELECT EXISTS(SELECT 1 FROM provisioning_jobs WHERE id = $id);";
+ exists.Parameters.AddWithValue("$id", id);
+ if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1)
+ {
+ return null;
+ }
+
+ var result = new List();
+ var command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT sequence, job_id, recorded_at, event_type, state,
+ step, progress_percent, message
+ FROM provisioning_events
+ WHERE job_id = $id
+ ORDER BY sequence DESC
+ LIMIT $limit;
+ """;
+ command.Parameters.AddWithValue("$id", id);
+ command.Parameters.AddWithValue("$limit", Math.Clamp(limit, 1, 200));
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ while (await reader.ReadAsync(cancellationToken))
+ {
+ result.Add(new ProvisioningEvent(
+ reader.GetInt64(0), reader.GetString(1), DateTimeOffset.Parse(reader.GetString(2)),
+ reader.GetString(3), reader.GetString(4), reader.GetString(5),
+ reader.GetInt32(6), reader.GetString(7)));
+ }
+ result.Reverse();
+ return result;
+ }
+
+ public async Task RetryProvisioningJobAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-retry:{actor}:{id}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
+ if (current is null)
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (current.State is not (ProvisioningJobStates.Failed
+ or ProvisioningJobStates.NeedsReconciliation)
+ || current.CurrentStep == "rollback-reconcile")
+ {
+ throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued);
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var originalLifetime = current.ExpiresAt - current.CreatedAt;
+ var retryLifetime = originalLifetime < TimeSpan.FromMinutes(5)
+ ? TimeSpan.FromMinutes(5)
+ : originalLifetime;
+ var updated = current with
+ {
+ State = ProvisioningJobStates.Queued,
+ ProgressPercent = 0,
+ CurrentStep = "retry-queued",
+ UpdatedAt = now,
+ ExpiresAt = now.Add(retryLifetime),
+ Version = current.Version + 1,
+ LastError = null
+ };
+ var update = connection.CreateCommand();
+ update.Transaction = transaction;
+ update.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = $state, progress_percent = 0, current_step = $step,
+ updated_at = $updated, expires_at = $expires,
+ version = $version, last_error = NULL
+ WHERE id = $id AND version = $previous_version;
+ """;
+ update.Parameters.AddWithValue("$state", updated.State);
+ update.Parameters.AddWithValue("$step", updated.CurrentStep);
+ update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O"));
+ update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O"));
+ update.Parameters.AddWithValue("$version", updated.Version);
+ update.Parameters.AddWithValue("$id", id);
+ update.Parameters.AddWithValue("$previous_version", current.Version);
+ if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+ {
+ throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued);
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, id, "job.retry.queued", updated.State,
+ "Operator queued a retry after reconciliation.", now, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, updated,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, actor, "provisioning.job.retry", id,
+ JsonSerializer.Serialize(new { request.Reason }), cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return updated;
+ }
+
+ public async Task StartProvisioningRollbackAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ string actor,
+ CancellationToken cancellationToken = default)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"provisioning-rollback:{actor}:{id}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
+ if (current is null)
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ if (current.State is not (ProvisioningJobStates.Failed
+ or ProvisioningJobStates.NeedsReconciliation
+ or ProvisioningJobStates.RollbackFailed))
+ {
+ throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack);
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var lifetime = current.ExpiresAt - current.CreatedAt;
+ if (lifetime < TimeSpan.FromMinutes(5))
+ {
+ lifetime = TimeSpan.FromMinutes(5);
+ }
+ var updated = current with
+ {
+ State = ProvisioningJobStates.RollingBack,
+ ProgressPercent = 0,
+ CurrentStep = "rollback-queued",
+ UpdatedAt = now,
+ ExpiresAt = now.Add(lifetime),
+ Version = current.Version + 1,
+ LastError = null
+ };
+ var update = connection.CreateCommand();
+ update.Transaction = transaction;
+ update.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = $state, progress_percent = 0, current_step = $step,
+ updated_at = $updated, expires_at = $expires,
+ version = $version, last_error = NULL
+ WHERE id = $id AND version = $previous_version;
+ """;
+ update.Parameters.AddWithValue("$state", updated.State);
+ update.Parameters.AddWithValue("$step", updated.CurrentStep);
+ update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O"));
+ update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O"));
+ update.Parameters.AddWithValue("$version", updated.Version);
+ update.Parameters.AddWithValue("$id", id);
+ update.Parameters.AddWithValue("$previous_version", current.Version);
+ if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+ {
+ throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack);
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, id, "job.rollback.queued", updated.State,
+ "Operator queued a rollback.", now, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, updated,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, actor, "provisioning.job.rollback", id,
+ JsonSerializer.Serialize(new { request.Reason }), cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return updated;
+ }
+
+ private async Task TransitionProvisioningJobAsync(
+ string id,
+ ProvisioningJobCommandRequest request,
+ string actor,
+ string? requiredState,
+ string targetState,
+ string eventType,
+ string auditAction,
+ bool setConfirmedAt,
+ CancellationToken cancellationToken)
+ {
+ await using var connection = await OpenAsync(cancellationToken);
+ await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+ var operationKey = $"{eventType}:{actor}:{id}:{request.IdempotencyKey}";
+ var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest);
+ var cached = await ReadIdempotentAsync(
+ connection, transaction, operationKey, requestHash,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ if (cached is not null)
+ {
+ await transaction.CommitAsync(cancellationToken);
+ return cached;
+ }
+
+ var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
+ if (current is null)
+ {
+ await transaction.RollbackAsync(cancellationToken);
+ return null;
+ }
+ var allowed = requiredState is not null
+ ? current.State == requiredState
+ : current.State is ProvisioningJobStates.Queued
+ or ProvisioningJobStates.Preflight
+ or ProvisioningJobStates.AwaitingConfirmation;
+ if (!allowed || current.ExpiresAt <= DateTimeOffset.UtcNow)
+ {
+ throw new ProvisioningTransitionException(current.State, targetState);
+ }
+
+ var now = DateTimeOffset.UtcNow;
+ var updated = current with
+ {
+ State = targetState,
+ CurrentStep = setConfirmedAt ? "confirmed-queued" : current.CurrentStep,
+ UpdatedAt = now,
+ ConfirmedAt = setConfirmedAt ? now : current.ConfirmedAt,
+ CancelledAt = targetState == ProvisioningJobStates.Cancelled ? now : current.CancelledAt,
+ Version = current.Version + 1
+ };
+ var update = connection.CreateCommand();
+ update.Transaction = transaction;
+ update.CommandText = """
+ UPDATE provisioning_jobs SET
+ state = $state, current_step = $step, updated_at = $updated, confirmed_at = $confirmed,
+ cancelled_at = $cancelled, version = $version
+ WHERE id = $id AND version = $previous_version;
+ """;
+ AddProvisioningJobParameters(update, updated);
+ update.Parameters.AddWithValue("$step", updated.CurrentStep);
+ update.Parameters.AddWithValue("$previous_version", current.Version);
+ if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+ {
+ throw new ProvisioningTransitionException(current.State, targetState);
+ }
+
+ await WriteProvisioningEventAsync(
+ connection, transaction, id, eventType, targetState,
+ $"Operator requested '{eventType}'.", now, cancellationToken);
+ await WriteIdempotentAsync(
+ connection, transaction, operationKey, requestHash, updated,
+ SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+ await WriteAuditAsync(
+ connection, transaction, actor, auditAction, id,
+ JsonSerializer.Serialize(new { request.Reason, State = targetState }), cancellationToken);
+ await transaction.CommitAsync(cancellationToken);
+ return updated;
+ }
+
+ private static async Task ReadProvisioningJobAsync(
+ SqliteConnection connection,
+ SqliteTransaction transaction,
+ string id,
+ CancellationToken cancellationToken)
+ {
+ var command = connection.CreateCommand();
+ command.Transaction = transaction;
+ command.CommandText = "SELECT * FROM provisioning_jobs WHERE id = $id;";
+ command.Parameters.AddWithValue("$id", id);
+ await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+ return await reader.ReadAsync(cancellationToken) ? ReadProvisioningJob(reader) : null;
+ }
+
+ private static ProvisioningJob ReadProvisioningJob(SqliteDataReader reader)
+ => new(
+ reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetInt32(3),
+ ParseParameters(reader.GetString(4)), reader.GetString(5),
+ reader.GetInt32(6) == 1, reader.GetString(7), reader.GetString(8),
+ DateTimeOffset.Parse(reader.GetString(9)), DateTimeOffset.Parse(reader.GetString(10)),
+ DateTimeOffset.Parse(reader.GetString(11)),
+ reader.IsDBNull(12) ? null : DateTimeOffset.Parse(reader.GetString(12)),
+ reader.IsDBNull(13) ? null : DateTimeOffset.Parse(reader.GetString(13)),
+ reader.GetInt64(14), reader.GetInt32(16), reader.GetString(17),
+ reader.IsDBNull(15) ? null : reader.GetString(15));
+
+ private static JsonElement ParseParameters(string json)
+ {
+ using var document = JsonDocument.Parse(json);
+ return document.RootElement.Clone();
+ }
+
+ private static void AddProvisioningJobParameters(SqliteCommand command, ProvisioningJob job)
+ {
+ command.Parameters.AddWithValue("$id", job.Id);
+ command.Parameters.AddWithValue("$node", job.NodeId);
+ command.Parameters.AddWithValue("$action", job.ActionType);
+ command.Parameters.AddWithValue("$schema", job.SchemaVersion);
+ command.Parameters.AddWithValue("$parameters", job.Parameters.GetRawText());
+ command.Parameters.AddWithValue("$state", job.State);
+ command.Parameters.AddWithValue("$confirmation", job.ConfirmationRequired ? 1 : 0);
+ command.Parameters.AddWithValue("$reason", job.AuditReason);
+ command.Parameters.AddWithValue("$actor", job.CreatedBy);
+ command.Parameters.AddWithValue("$created", job.CreatedAt.ToString("O"));
+ command.Parameters.AddWithValue("$updated", job.UpdatedAt.ToString("O"));
+ command.Parameters.AddWithValue("$expires", job.ExpiresAt.ToString("O"));
+ command.Parameters.AddWithValue("$confirmed", job.ConfirmedAt is null ? DBNull.Value : job.ConfirmedAt.Value.ToString("O"));
+ command.Parameters.AddWithValue("$cancelled", job.CancelledAt is null ? DBNull.Value : job.CancelledAt.Value.ToString("O"));
+ command.Parameters.AddWithValue("$version", job.Version);
+ }
+
+ private static async Task WriteProvisioningEventAsync(
+ SqliteConnection connection,
+ SqliteTransaction transaction,
+ string jobId,
+ string eventType,
+ string state,
+ string message,
+ DateTimeOffset recordedAt,
+ CancellationToken cancellationToken)
+ {
+ var command = connection.CreateCommand();
+ command.Transaction = transaction;
+ command.CommandText = """
+ INSERT INTO provisioning_events(
+ job_id, recorded_at, event_type, state, message, step, progress_percent)
+ SELECT $job, $recorded, $event, $state, $message,
+ current_step, progress_percent
+ FROM provisioning_jobs
+ WHERE id = $job;
+ """;
+ command.Parameters.AddWithValue("$job", jobId);
+ command.Parameters.AddWithValue("$recorded", recordedAt.ToString("O"));
+ command.Parameters.AddWithValue("$event", eventType);
+ command.Parameters.AddWithValue("$state", state);
+ command.Parameters.AddWithValue("$message", message);
+ await command.ExecuteNonQueryAsync(cancellationToken);
+ }
+}
+
+public sealed class ProvisioningNodeNotFoundException(string nodeId) : Exception($"Node '{nodeId}' was not found.");
+
+public sealed class ProvisioningTransitionException(string state, string targetState)
+ : Exception($"Provisioning job cannot transition from '{state}' to '{targetState}'.");
+
+internal static class ProvisioningStateMachine
+{
+ public static bool CanReport(string current, string target, int currentProgress, int targetProgress)
+ {
+ if (targetProgress < currentProgress || targetProgress is < 0 or > 100)
+ {
+ return false;
+ }
+ if (target is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation)
+ {
+ return current is ProvisioningJobStates.Preflight
+ or ProvisioningJobStates.Running
+ or ProvisioningJobStates.Verifying;
+ }
+ if (target == ProvisioningJobStates.RollbackFailed)
+ {
+ return current == ProvisioningJobStates.RollingBack;
+ }
+ return (current, target) switch
+ {
+ (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true,
+ (ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true,
+ (ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true,
+ (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true,
+ (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true,
+ (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100,
+ (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true,
+ (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100,
+ _ => false
+ };
+ }
+}
diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs
index 42f1101..c0456db 100644
--- a/src/ServerMonitorManager.Core/Contracts.cs
+++ b/src/ServerMonitorManager.Core/Contracts.cs
@@ -1,3 +1,6 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
namespace ServerMonitorManager.Core;
public sealed record EnrollmentRequest(
@@ -140,3 +143,330 @@ public sealed record ControlEvent(
string PayloadJson);
public sealed record ControlError(string Error);
+
+public sealed record ProvisioningJobCreateRequest(
+ string ActionType,
+ int SchemaVersion,
+ JsonElement Parameters,
+ int TtlMinutes,
+ string AuditReason,
+ string IdempotencyKey);
+
+public sealed record ProvisioningJobCommandRequest(
+ string Reason,
+ string IdempotencyKey);
+
+public sealed record ProvisioningJobProgressRequest(
+ string State,
+ int ProgressPercent,
+ string Step,
+ string EventCode,
+ string Message,
+ string IdempotencyKey);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record ProvisioningHelperRequest(
+ string ProtocolVersion,
+ string JobId,
+ string ActionType,
+ int SchemaVersion,
+ string ModuleHash,
+ JsonElement Parameters);
+
+public sealed record ProvisioningHelperResponse(
+ bool Success,
+ string Code,
+ string Message,
+ ProvisioningPreflightResult? Preflight,
+ SystemBaseInstallPlan? BaseInstallPlan);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record ProvisioningPreflightResult(
+ string OperatingSystem,
+ string OperatingSystemVersion,
+ string Architecture,
+ bool HasSystemd,
+ bool HasSshd,
+ bool HasNftables,
+ bool HasWireGuard,
+ bool HasApt);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record ProvisioningPreflightReportRequest(
+ ProvisioningPreflightResult Facts,
+ DateTimeOffset ObservedAt,
+ string IdempotencyKey);
+
+public sealed record NodePreflightFacts(
+ string NodeId,
+ int SchemaVersion,
+ ProvisioningPreflightResult Facts,
+ DateTimeOffset ObservedAt,
+ string SourceJobId,
+ DateTimeOffset UpdatedAt);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record PreflightDesiredRequirements(
+ bool RequireSystemd,
+ bool RequireSshd,
+ bool RequireNftables,
+ bool RequireWireGuard,
+ bool RequireApt,
+ string[] AllowedArchitectures);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record PreflightDesiredStateUpdateRequest(
+ int SchemaVersion,
+ PreflightDesiredRequirements Desired,
+ string AuditReason,
+ string IdempotencyKey);
+
+public sealed record NodePreflightDesiredState(
+ string NodeId,
+ int SchemaVersion,
+ PreflightDesiredRequirements Desired,
+ long Version,
+ string UpdatedBy,
+ string AuditReason,
+ DateTimeOffset UpdatedAt);
+
+public sealed record PreflightDriftAssessment(
+ string NodeId,
+ string Status,
+ string[] DriftCodes,
+ NodePreflightDesiredState? Desired,
+ NodePreflightFacts? Facts);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record SystemBaseInstallParameters(
+ string Timezone,
+ string Locale,
+ bool AptUpdate,
+ bool AptUpgrade,
+ int PackageCatalogVersion,
+ string[] PackageGroupIds,
+ string SwapMode,
+ int? SwapSizeMiB,
+ int VmSwappiness,
+ bool EnableUnattendedUpgrades,
+ string RebootPolicy);
+
+public sealed record SystemPackageGroup(
+ string Id,
+ string[] Packages);
+
+public sealed record SystemBaseInstallCatalog(
+ int Version,
+ SystemPackageGroup[] Groups);
+
+public sealed record SystemBaseInstallPlan(
+ string Timezone,
+ string Locale,
+ bool AptUpdate,
+ bool AptUpgrade,
+ string[] Packages,
+ string SwapMode,
+ int? SwapSizeMiB,
+ int VmSwappiness,
+ bool EnableUnattendedUpgrades,
+ string RebootPolicy,
+ string[] Warnings);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record SystemBaseInstallPlanReportRequest(
+ SystemBaseInstallPlan Plan,
+ string IdempotencyKey);
+
+public sealed record ProvisioningBaseInstallPlanRecord(
+ string JobId,
+ string NodeId,
+ int SchemaVersion,
+ SystemBaseInstallPlan Plan,
+ DateTimeOffset CreatedAt);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record ProvisioningExecutionGrant(
+ string ProtocolVersion,
+ string JobId,
+ string NodeId,
+ string ActionType,
+ int SchemaVersion,
+ string PlanSha256,
+ long IssuedAtUnixSeconds,
+ long ExpiresAtUnixSeconds,
+ string Nonce,
+ string SignatureAlgorithm,
+ string Signature);
+
+[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
+public sealed record ProvisioningExecutionGrantRequest(string IdempotencyKey);
+
+public static class PreflightDriftStatuses
+{
+ public const string NotConfigured = "NotConfigured";
+ public const string Unknown = "Unknown";
+ public const string InSync = "InSync";
+ public const string Drifted = "Drifted";
+}
+
+public static class PreflightDriftCodes
+{
+ public const string FactsMissing = "facts.missing";
+ public const string SystemdMissing = "systemd.missing";
+ public const string SshdMissing = "sshd.missing";
+ public const string NftablesMissing = "nftables.missing";
+ public const string WireGuardMissing = "wireguard.missing";
+ public const string AptMissing = "apt.missing";
+ public const string ArchitectureUnsupported = "architecture.unsupported";
+}
+
+public sealed record ProvisioningJob(
+ string Id,
+ string NodeId,
+ string ActionType,
+ int SchemaVersion,
+ JsonElement Parameters,
+ string State,
+ bool ConfirmationRequired,
+ string AuditReason,
+ string CreatedBy,
+ DateTimeOffset CreatedAt,
+ DateTimeOffset UpdatedAt,
+ DateTimeOffset ExpiresAt,
+ DateTimeOffset? ConfirmedAt,
+ DateTimeOffset? CancelledAt,
+ long Version,
+ int ProgressPercent,
+ string CurrentStep,
+ string? LastError);
+
+public sealed record ProvisioningEvent(
+ long Sequence,
+ string JobId,
+ DateTimeOffset RecordedAt,
+ string EventType,
+ string State,
+ string Step,
+ int ProgressPercent,
+ string Message);
+
+public static class ProvisioningJobStates
+{
+ public const string Queued = "Queued";
+ public const string Preflight = "Preflight";
+ public const string AwaitingConfirmation = "AwaitingConfirmation";
+ public const string Running = "Running";
+ public const string Verifying = "Verifying";
+ public const string Completed = "Completed";
+ public const string Failed = "Failed";
+ public const string NeedsReconciliation = "NeedsReconciliation";
+ public const string RollingBack = "RollingBack";
+ public const string RolledBack = "RolledBack";
+ public const string RollbackFailed = "RollbackFailed";
+ public const string Cancelled = "Cancelled";
+}
+
+public static class ProvisioningActionCatalog
+{
+ public const string PreflightModuleHash =
+ "2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b";
+ public const string SystemBaseInstallModuleHash =
+ "355d55e214b941160a32957ced1a681e3c7324f94ecb340f26042f0c3b59b99e";
+}
+
+public static class SystemBaseInstallCatalogDefinition
+{
+ public const int Version = 1;
+
+ public static SystemBaseInstallCatalog Create()
+ => new(Version,
+ [
+ new("core", ["ca-certificates", "curl", "jq"]),
+ new("development", ["build-essential", "git"]),
+ new("diagnostics", ["htop", "iotop"]),
+ new("container-host", ["dbus-user-session", "uidmap"])
+ ]);
+
+ public static bool ContainsGroup(string id)
+ => id is "core" or "development" or "diagnostics" or "container-host";
+
+ public static string[] ExpandGroups(IEnumerable ids)
+ {
+ var selected = ids.ToHashSet(StringComparer.Ordinal);
+ return Create().Groups
+ .Where(group => selected.Contains(group.Id))
+ .SelectMany(group => group.Packages)
+ .Distinct(StringComparer.Ordinal)
+ .ToArray();
+ }
+}
+
+public static class SystemBaseInstallSchema
+{
+ private static readonly HashSet AllowedPlanWarnings =
+ new(["apt.missing", "timezone.missing"], StringComparer.Ordinal);
+
+ public static bool TryParse(JsonElement json, out SystemBaseInstallParameters? parameters)
+ {
+ try
+ {
+ parameters = JsonSerializer.Deserialize(
+ json, SmmJsonContext.Default.SystemBaseInstallParameters);
+ return parameters is not null && IsValid(parameters);
+ }
+ catch (JsonException)
+ {
+ parameters = null;
+ return false;
+ }
+ }
+
+ public static bool IsValid(SystemBaseInstallParameters parameters)
+ => IsSafeTimezone(parameters.Timezone)
+ && IsSafeLocale(parameters.Locale)
+ && (!parameters.AptUpgrade || parameters.AptUpdate)
+ && parameters.PackageCatalogVersion == SystemBaseInstallCatalogDefinition.Version
+ && parameters.PackageGroupIds is { Length: <= 4 }
+ && parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count()
+ == parameters.PackageGroupIds.Length
+ && parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup)
+ && parameters.SwapMode is "disabled" or "automatic" or "explicit"
+ && (parameters.SwapMode == "explicit"
+ ? parameters.SwapSizeMiB is >= 128 and <= 1_048_576
+ : parameters.SwapSizeMiB is null)
+ && parameters.VmSwappiness is >= 0 and <= 200
+ && parameters.RebootPolicy == "never";
+
+ public static bool IsValidPlan(
+ SystemBaseInstallParameters parameters,
+ SystemBaseInstallPlan plan)
+ => plan is not null
+ && string.Equals(plan.Timezone, parameters.Timezone, StringComparison.Ordinal)
+ && string.Equals(plan.Locale, parameters.Locale, StringComparison.Ordinal)
+ && plan.AptUpdate == parameters.AptUpdate
+ && plan.AptUpgrade == parameters.AptUpgrade
+ && plan.Packages is not null
+ && plan.Packages.SequenceEqual(
+ SystemBaseInstallCatalogDefinition.ExpandGroups(parameters.PackageGroupIds),
+ StringComparer.Ordinal)
+ && string.Equals(plan.SwapMode, parameters.SwapMode, StringComparison.Ordinal)
+ && plan.SwapSizeMiB == parameters.SwapSizeMiB
+ && plan.VmSwappiness == parameters.VmSwappiness
+ && plan.EnableUnattendedUpgrades == parameters.EnableUnattendedUpgrades
+ && string.Equals(plan.RebootPolicy, parameters.RebootPolicy, StringComparison.Ordinal)
+ && plan.Warnings is { Length: <= 2 }
+ && plan.Warnings.Distinct(StringComparer.Ordinal).Count() == plan.Warnings.Length
+ && plan.Warnings.All(AllowedPlanWarnings.Contains);
+
+ private static bool IsSafeTimezone(string? value)
+ => value is { Length: >= 1 and <= 64 }
+ && value[0] is not '/' and not '.'
+ && !value.Contains("..", StringComparison.Ordinal)
+ && value.All(character => char.IsAsciiLetterOrDigit(character)
+ || character is '/' or '_' or '-' or '+');
+
+ private static bool IsSafeLocale(string? value)
+ => value is { Length: >= 1 and <= 32 }
+ && value.All(character => char.IsAsciiLetterOrDigit(character)
+ || character is '_' or '-' or '.' or '@');
+}
diff --git a/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs
new file mode 100644
index 0000000..e97e9eb
--- /dev/null
+++ b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs
@@ -0,0 +1,134 @@
+using System.Security.Cryptography;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Text.Json;
+
+namespace ServerMonitorManager.Core;
+
+public static class ProvisioningExecutionGrantCodec
+{
+ public const string ProtocolVersion = "1";
+ public const string SignatureAlgorithm = "ECDSA-P256-SHA256-P1363";
+ public static readonly TimeSpan MaximumLifetime = TimeSpan.FromMinutes(5);
+
+ public static string ComputePlanSha256(SystemBaseInstallPlan plan)
+ {
+ var json = JsonSerializer.SerializeToUtf8Bytes(
+ plan, SmmJsonContext.Default.SystemBaseInstallPlan);
+ return Convert.ToHexStringLower(SHA256.HashData(json));
+ }
+
+ public static byte[] CreateSigningPayload(ProvisioningExecutionGrant grant)
+ => Encoding.UTF8.GetBytes(string.Join('\n',
+ [
+ "SMM-PROVISIONING-GRANT-V1",
+ grant.ProtocolVersion,
+ grant.JobId,
+ grant.NodeId,
+ grant.ActionType,
+ grant.SchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture),
+ grant.PlanSha256,
+ grant.IssuedAtUnixSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture),
+ grant.ExpiresAtUnixSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture),
+ grant.Nonce,
+ grant.SignatureAlgorithm
+ ]));
+
+ public static bool Verify(
+ ProvisioningExecutionGrant grant,
+ X509Certificate2 controlAuthority,
+ string expectedJobId,
+ string expectedNodeId,
+ SystemBaseInstallPlan expectedPlan,
+ DateTimeOffset now)
+ {
+ if (grant.ProtocolVersion != ProtocolVersion
+ || grant.SignatureAlgorithm != SignatureAlgorithm
+ || grant.JobId != expectedJobId
+ || grant.NodeId != expectedNodeId
+ || grant.ActionType != "system.base-install"
+ || grant.SchemaVersion != 1
+ || grant.JobId is not { Length: 32 }
+ || !grant.JobId.All(Uri.IsHexDigit)
+ || grant.Nonce is not { Length: 32 }
+ || !grant.Nonce.All(Uri.IsHexDigit)
+ || grant.PlanSha256 is not { Length: 64 }
+ || !grant.PlanSha256.All(Uri.IsHexDigit)
+ || grant.Signature is not { Length: >= 1 and <= 128 })
+ {
+ return false;
+ }
+
+ DateTimeOffset issuedAt;
+ DateTimeOffset expiresAt;
+ try
+ {
+ issuedAt = DateTimeOffset.FromUnixTimeSeconds(grant.IssuedAtUnixSeconds);
+ expiresAt = DateTimeOffset.FromUnixTimeSeconds(grant.ExpiresAtUnixSeconds);
+ }
+ catch (ArgumentOutOfRangeException)
+ {
+ return false;
+ }
+ if (issuedAt > now.AddSeconds(30)
+ || expiresAt <= now
+ || expiresAt <= issuedAt
+ || expiresAt - issuedAt > MaximumLifetime)
+ {
+ return false;
+ }
+
+ var expectedHash = ComputePlanSha256(expectedPlan);
+ if (!CryptographicOperations.FixedTimeEquals(
+ Encoding.ASCII.GetBytes(expectedHash),
+ Encoding.ASCII.GetBytes(grant.PlanSha256.ToLowerInvariant())))
+ {
+ return false;
+ }
+
+ byte[] signature;
+ try
+ {
+ signature = DecodeBase64Url(grant.Signature);
+ }
+ catch (FormatException)
+ {
+ return false;
+ }
+ if (signature.Length != 64)
+ {
+ return false;
+ }
+
+ using var key = controlAuthority.GetECDsaPublicKey();
+ return key is { KeySize: 256 }
+ && key.VerifyData(
+ CreateSigningPayload(grant), signature, HashAlgorithmName.SHA256,
+ DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
+ }
+
+ public static string EncodeBase64Url(ReadOnlySpan value)
+ => Convert.ToBase64String(value)
+ .TrimEnd('=')
+ .Replace('+', '-')
+ .Replace('/', '_');
+
+ private static byte[] DecodeBase64Url(string value)
+ {
+ if (value.Length is < 1 or > 128
+ || value.Any(character => !char.IsAsciiLetterOrDigit(character)
+ && character is not '-' and not '_'))
+ {
+ throw new FormatException("Invalid base64url value.");
+ }
+ var padded = value.Replace('-', '+').Replace('_', '/');
+ padded += (padded.Length % 4) switch
+ {
+ 2 => "==",
+ 3 => "=",
+ 0 => string.Empty,
+ _ => throw new FormatException("Invalid base64url value.")
+ };
+ return Convert.FromBase64String(padded);
+ }
+}
diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs
index 847bd3b..88e6a38 100644
--- a/src/ServerMonitorManager.Core/SmmJsonContext.cs
+++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs
@@ -2,6 +2,7 @@ using System.Text.Json.Serialization;
namespace ServerMonitorManager.Core;
+[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)]
[JsonSerializable(typeof(EnrollmentRequest))]
[JsonSerializable(typeof(EnrollmentResponse))]
[JsonSerializable(typeof(AgentHeartbeat))]
@@ -25,4 +26,29 @@ namespace ServerMonitorManager.Core;
[JsonSerializable(typeof(LinkPolicy[]))]
[JsonSerializable(typeof(ControlEvent))]
[JsonSerializable(typeof(ControlError))]
+[JsonSerializable(typeof(ProvisioningJobCreateRequest))]
+[JsonSerializable(typeof(ProvisioningJobCommandRequest))]
+[JsonSerializable(typeof(ProvisioningJobProgressRequest))]
+[JsonSerializable(typeof(ProvisioningHelperRequest))]
+[JsonSerializable(typeof(ProvisioningHelperResponse))]
+[JsonSerializable(typeof(ProvisioningPreflightResult))]
+[JsonSerializable(typeof(ProvisioningPreflightReportRequest))]
+[JsonSerializable(typeof(NodePreflightFacts))]
+[JsonSerializable(typeof(PreflightDesiredRequirements))]
+[JsonSerializable(typeof(PreflightDesiredStateUpdateRequest))]
+[JsonSerializable(typeof(NodePreflightDesiredState))]
+[JsonSerializable(typeof(PreflightDriftAssessment))]
+[JsonSerializable(typeof(SystemBaseInstallParameters))]
+[JsonSerializable(typeof(SystemPackageGroup))]
+[JsonSerializable(typeof(SystemPackageGroup[]))]
+[JsonSerializable(typeof(SystemBaseInstallCatalog))]
+[JsonSerializable(typeof(SystemBaseInstallPlan))]
+[JsonSerializable(typeof(SystemBaseInstallPlanReportRequest))]
+[JsonSerializable(typeof(ProvisioningBaseInstallPlanRecord))]
+[JsonSerializable(typeof(ProvisioningExecutionGrant))]
+[JsonSerializable(typeof(ProvisioningExecutionGrantRequest))]
+[JsonSerializable(typeof(ProvisioningJob))]
+[JsonSerializable(typeof(ProvisioningJob[]))]
+[JsonSerializable(typeof(ProvisioningEvent))]
+[JsonSerializable(typeof(ProvisioningEvent[]))]
public sealed partial class SmmJsonContext : JsonSerializerContext;
diff --git a/src/ServerMonitorManager.Provisioning.Helper/Program.cs b/src/ServerMonitorManager.Provisioning.Helper/Program.cs
new file mode 100644
index 0000000..0407ecc
--- /dev/null
+++ b/src/ServerMonitorManager.Provisioning.Helper/Program.cs
@@ -0,0 +1,24 @@
+using ServerMonitorManager.Provisioning.Helper;
+
+if (!OperatingSystem.IsLinux())
+{
+ Console.Error.WriteLine("The provisioning helper is supported only on Linux.");
+ return 2;
+}
+
+if (args.Length != 0)
+{
+ Console.Error.WriteLine("The provisioning helper does not accept command-line arguments.");
+ return 2;
+}
+const string socketPath = "/run/ochenstarik-server-monitor-manager/provisioning.sock";
+
+using var shutdown = new CancellationTokenSource();
+Console.CancelKeyPress += (_, eventArgs) =>
+{
+ eventArgs.Cancel = true;
+ shutdown.Cancel();
+};
+
+await new ProvisioningHelperServer(socketPath).RunAsync(shutdown.Token);
+return 0;
diff --git a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs
new file mode 100644
index 0000000..b1c0455
--- /dev/null
+++ b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs
@@ -0,0 +1,186 @@
+using System.Net.Sockets;
+using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
+using System.Text;
+using System.Text.Json;
+using ServerMonitorManager.Core;
+
+namespace ServerMonitorManager.Provisioning.Helper;
+
+public sealed class ProvisioningHelperServer(string socketPath)
+{
+ private const int MaximumRequestBytes = 16 * 1024;
+
+ [SupportedOSPlatform("linux")]
+ public async Task RunAsync(CancellationToken cancellationToken)
+ {
+ Directory.CreateDirectory(Path.GetDirectoryName(socketPath)!);
+ File.Delete(socketPath);
+ using var listener = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified);
+ listener.Bind(new UnixDomainSocketEndPoint(socketPath));
+ File.SetUnixFileMode(socketPath,
+ UnixFileMode.UserRead | UnixFileMode.UserWrite
+ | UnixFileMode.GroupRead | UnixFileMode.GroupWrite);
+ listener.Listen(8);
+ try
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ var connection = await listener.AcceptAsync(cancellationToken);
+ await HandleAsync(connection, cancellationToken);
+ }
+ }
+ finally
+ {
+ File.Delete(socketPath);
+ }
+ }
+
+ public static ProvisioningHelperResponse Execute(ProvisioningHelperRequest request)
+ {
+ if (request.ProtocolVersion != "1")
+ {
+ return Failure("protocol.unsupported", "Unsupported helper protocol version.");
+ }
+ if (request.JobId.Length != 32 || !request.JobId.All(Uri.IsHexDigit))
+ {
+ return Failure("request.invalid-job", "Invalid provisioning job identifier.");
+ }
+ if (request.SchemaVersion != 1 || request.Parameters.ValueKind != JsonValueKind.Object)
+ {
+ return Failure("action.denied", "The requested action is not allowed.");
+ }
+
+ return request.ActionType switch
+ {
+ "preflight" => ExecutePreflight(request),
+ "system.base-install" => CreateBaseInstallPlan(request),
+ _ => Failure("action.denied", "The requested action is not allowed.")
+ };
+ }
+
+ private static ProvisioningHelperResponse ExecutePreflight(ProvisioningHelperRequest request)
+ {
+ if (request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash
+ || request.Parameters.EnumerateObject().Any())
+ {
+ return Failure("action.denied", "The requested action is not allowed.");
+ }
+
+ var release = ReadOperatingSystemRelease();
+ var result = new ProvisioningPreflightResult(
+ release.GetValueOrDefault("ID", "linux"),
+ release.GetValueOrDefault("VERSION_ID", "unknown"),
+ RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(),
+ Directory.Exists("/run/systemd/system"),
+ Exists("/usr/sbin/sshd", "/usr/bin/sshd", "/sbin/sshd"),
+ Exists("/usr/sbin/nft", "/usr/bin/nft", "/sbin/nft"),
+ Exists("/usr/bin/wg", "/usr/sbin/wg", "/bin/wg"),
+ Exists("/usr/bin/apt-get", "/bin/apt-get"));
+ return new ProvisioningHelperResponse(
+ true, "preflight.completed", "Preflight completed.", result, null);
+ }
+
+ private static ProvisioningHelperResponse CreateBaseInstallPlan(ProvisioningHelperRequest request)
+ {
+ if (request.ModuleHash != ProvisioningActionCatalog.SystemBaseInstallModuleHash
+ || !SystemBaseInstallSchema.TryParse(request.Parameters, out var parameters))
+ {
+ return Failure("action.denied", "The requested action is not allowed.");
+ }
+
+ var warnings = new List();
+ if (!Exists("/usr/bin/apt-get", "/bin/apt-get"))
+ {
+ warnings.Add("apt.missing");
+ }
+ if (!File.Exists(Path.Combine("/usr/share/zoneinfo", parameters!.Timezone)))
+ {
+ warnings.Add("timezone.missing");
+ }
+ var plan = new SystemBaseInstallPlan(
+ parameters.Timezone,
+ parameters.Locale,
+ parameters.AptUpdate,
+ parameters.AptUpgrade,
+ SystemBaseInstallCatalogDefinition.ExpandGroups(parameters.PackageGroupIds),
+ parameters.SwapMode,
+ parameters.SwapSizeMiB,
+ parameters.VmSwappiness,
+ parameters.EnableUnattendedUpgrades,
+ parameters.RebootPolicy,
+ [.. warnings]);
+ return new ProvisioningHelperResponse(
+ true, "system.base-install.plan-ready", "Base install plan is ready.", null, plan);
+ }
+
+ private static async Task HandleAsync(Socket socket, CancellationToken cancellationToken)
+ {
+ using (socket)
+ await using (var stream = new NetworkStream(socket, ownsSocket: false))
+ {
+ ProvisioningHelperResponse response;
+ try
+ {
+ var payload = await ReadRequestAsync(stream, cancellationToken);
+ var request = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperRequest)
+ ?? throw new JsonException("Empty request.");
+ response = Execute(request);
+ }
+ catch (Exception exception) when (exception is JsonException or InvalidDataException)
+ {
+ response = Failure("request.invalid", "Invalid helper request.");
+ }
+ var json = JsonSerializer.Serialize(response, SmmJsonContext.Default.ProvisioningHelperResponse) + "\n";
+ await stream.WriteAsync(Encoding.UTF8.GetBytes(json), cancellationToken);
+ }
+ }
+
+ private static async Task ReadRequestAsync(Stream stream, CancellationToken cancellationToken)
+ {
+ using var buffer = new MemoryStream();
+ var singleByte = new byte[1];
+ while (buffer.Length <= MaximumRequestBytes)
+ {
+ var count = await stream.ReadAsync(singleByte, cancellationToken);
+ if (count == 0 || singleByte[0] == (byte)'\n')
+ {
+ break;
+ }
+ buffer.WriteByte(singleByte[0]);
+ }
+ if (buffer.Length == 0 || buffer.Length > MaximumRequestBytes)
+ {
+ throw new InvalidDataException("Request size is invalid.");
+ }
+ return buffer.ToArray();
+ }
+
+ private static Dictionary ReadOperatingSystemRelease()
+ {
+ var result = new Dictionary(StringComparer.Ordinal);
+ if (!File.Exists("/etc/os-release"))
+ {
+ return result;
+ }
+ foreach (var line in File.ReadLines("/etc/os-release"))
+ {
+ var separator = line.IndexOf('=');
+ if (separator <= 0)
+ {
+ continue;
+ }
+ var key = line[..separator];
+ if (key is "ID" or "VERSION_ID")
+ {
+ result[key] = line[(separator + 1)..].Trim().Trim('"');
+ }
+ }
+ return result;
+ }
+
+ private static bool Exists(params string[] paths) => paths.Any(File.Exists);
+
+ private static ProvisioningHelperResponse Failure(string code, string message)
+ => new(false, code, message, null, null);
+}
diff --git a/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj b/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj
new file mode 100644
index 0000000..749100f
--- /dev/null
+++ b/src/ServerMonitorManager.Provisioning.Helper/ServerMonitorManager.Provisioning.Helper.csproj
@@ -0,0 +1,14 @@
+
+
+ Exe
+ net10.0
+ enable
+ enable
+ true
+ 0.1.0
+ ochenstarik-smm-provisioning-helper
+
+
+
+
+
diff --git a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs
index ceda103..210f2ac 100644
--- a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs
@@ -2,6 +2,7 @@ using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using Microsoft.Extensions.Options;
using ServerMonitorManager.Control;
+using ServerMonitorManager.Core;
using Xunit;
namespace ServerMonitorManager.Control.Tests;
@@ -49,6 +50,59 @@ public sealed class CertificateAuthorityTests : IDisposable
oid => oid.Value == "1.3.6.1.5.5.7.3.2");
}
+ [Fact]
+ public void ProvisioningExecutionGrantIsBoundToConfirmedJobPlanAndExpiry()
+ {
+ Directory.CreateDirectory(_directory);
+ var caPath = Path.Combine(_directory, "grant-control-ca.pfx");
+ using var caKey = ECDsa.Create(ECCurve.NamedCurves.nistP256);
+ var caRequest = new CertificateRequest("CN=SMM Grant Test CA", caKey, HashAlgorithmName.SHA256);
+ caRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true));
+ caRequest.CertificateExtensions.Add(new X509KeyUsageExtension(
+ X509KeyUsageFlags.KeyCertSign | X509KeyUsageFlags.CrlSign | X509KeyUsageFlags.DigitalSignature,
+ true));
+ using var ca = caRequest.CreateSelfSigned(
+ DateTimeOffset.UtcNow.AddMinutes(-1),
+ DateTimeOffset.UtcNow.AddYears(2));
+ File.WriteAllBytes(caPath, ca.Export(X509ContentType.Pfx));
+ using var authority = new CertificateAuthority(Options.Create(new ControlOptions
+ {
+ DatabasePath = Path.Combine(_directory, "unused-grant.db"),
+ CertificateAuthorityPath = caPath
+ }));
+
+ var now = DateTimeOffset.UtcNow;
+ var parameters = System.Text.Json.JsonSerializer.SerializeToElement(
+ new SystemBaseInstallParameters(
+ "UTC", "en_US.UTF-8", true, false, 1, ["core"],
+ "disabled", null, 60, true, "never"),
+ SmmJsonContext.Default.SystemBaseInstallParameters);
+ var job = new ProvisioningJob(
+ Guid.NewGuid().ToString("N"), "home", "system.base-install", 1, parameters,
+ ProvisioningJobStates.Queued, true, "test", "operator",
+ now.AddMinutes(-1), now, now.AddMinutes(30), now, null, 4, 25,
+ "confirmed-queued", null);
+ var plan = new SystemBaseInstallPlan(
+ "UTC", "en_US.UTF-8", true, false,
+ SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]),
+ "disabled", null, 60, true, "never", []);
+
+ var grant = authority.SignProvisioningExecutionGrant(
+ job, plan, now, TimeSpan.FromMinutes(2));
+
+ Assert.True(ProvisioningExecutionGrantCodec.Verify(
+ grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now));
+ Assert.False(ProvisioningExecutionGrantCodec.Verify(
+ grant, authority.PublicCertificate, job.Id, "other", plan, now));
+ Assert.False(ProvisioningExecutionGrantCodec.Verify(
+ grant, authority.PublicCertificate, job.Id, job.NodeId,
+ plan with { Packages = [.. plan.Packages, "untrusted-package"] }, now));
+ Assert.False(ProvisioningExecutionGrantCodec.Verify(
+ grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now.AddMinutes(3)));
+ Assert.Throws(() => authority.SignProvisioningExecutionGrant(
+ job with { ConfirmedAt = null }, plan, now, TimeSpan.FromMinutes(2)));
+ }
+
public void Dispose()
{
if (Directory.Exists(_directory))
diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs
index 0c6096b..0c9531a 100644
--- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs
@@ -76,6 +76,359 @@ public sealed class ControlApiTests : IAsyncDisposable
Assert.Equal("application/problem+json", response.Content.Headers.ContentType?.MediaType);
}
+ [Fact]
+ public async Task OperatorCanCreateAndReadProvisioningJob()
+ {
+ var store = _factory.Services.GetRequiredService();
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var token = await store.CreateEnrollmentTokenAsync("home", TimeSpan.FromMinutes(10), cancellationToken);
+ await store.EnrollAsync(
+ new ServerMonitorManager.Core.EnrollmentRequest(
+ "home", token, "csr", Guid.NewGuid().ToString()),
+ () => new ServerMonitorManager.Control.IssuedCertificate(
+ "certificate", "ca", "F1E2", DateTimeOffset.UtcNow.AddDays(1)),
+ cancellationToken);
+
+ using var anonymous = _factory.CreateClient();
+ Assert.Equal(HttpStatusCode.Unauthorized, (await anonymous.GetAsync(
+ $"/api/v1/control/provisioning/jobs/{Guid.NewGuid():N}", cancellationToken)).StatusCode);
+
+ using var client = _factory.CreateClient();
+ client.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc");
+ client.DefaultRequestHeaders.Add("X-Test-Role", "Operator");
+ var catalog = await client.GetFromJsonAsync(
+ "/api/v1/control/provisioning/catalogs/system-base-install/1",
+ cancellationToken);
+ Assert.Equal(1, catalog!.Version);
+ Assert.Contains(catalog.Groups, group => group.Id == "development"
+ && group.Packages.Contains("git"));
+ using var response = await client.PostAsJsonAsync(
+ "/api/v1/control/agents/home/provisioning/jobs",
+ new
+ {
+ actionType = "system.base-install",
+ schemaVersion = 1,
+ parameters = new
+ {
+ timezone = "UTC",
+ locale = "en_US.UTF-8",
+ aptUpdate = true,
+ aptUpgrade = false,
+ packageCatalogVersion = 1,
+ packageGroupIds = new[] { "core", "development" },
+ swapMode = "automatic",
+ swapSizeMiB = (int?)null,
+ vmSwappiness = 60,
+ enableUnattendedUpgrades = true,
+ rebootPolicy = "never"
+ },
+ ttlMinutes = 60,
+ auditReason = "API integration test",
+ idempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken);
+
+ Assert.Equal(HttpStatusCode.Created, response.StatusCode);
+ var job = await response.Content.ReadFromJsonAsync(
+ cancellationToken);
+ Assert.NotNull(job);
+ Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, job.State);
+ Assert.Equal(HttpStatusCode.Conflict, (await client.PostAsJsonAsync(
+ $"/api/v1/control/provisioning/jobs/{job.Id}/confirm",
+ new { reason = "Too early", idempotencyKey = Guid.NewGuid().ToString() },
+ cancellationToken)).StatusCode);
+
+ using var agent = _factory.CreateClient();
+ agent.DefaultRequestHeaders.Add("X-Test-Identity", "home");
+ agent.DefaultRequestHeaders.Add("X-Test-Role", "Agent");
+ var claimed = await agent.GetFromJsonAsync(
+ "/api/v1/agents/provisioning/jobs/next", cancellationToken);
+ Assert.Equal(job.Id, claimed!.Id);
+ Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State);
+ var expectedPackages = ServerMonitorManager.Core.SystemBaseInstallCatalogDefinition
+ .ExpandGroups(["core", "development"]);
+ Assert.Equal(HttpStatusCode.OK, (await agent.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{job.Id}/base-install-plan",
+ new
+ {
+ plan = new
+ {
+ timezone = "UTC",
+ locale = "en_US.UTF-8",
+ aptUpdate = true,
+ aptUpgrade = false,
+ packages = expectedPackages,
+ swapMode = "automatic",
+ swapSizeMiB = (int?)null,
+ vmSwappiness = 60,
+ enableUnattendedUpgrades = true,
+ rebootPolicy = "never",
+ warnings = Array.Empty()
+ },
+ idempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken)).StatusCode);
+ var storedPlan = await client.GetFromJsonAsync<
+ ServerMonitorManager.Core.ProvisioningBaseInstallPlanRecord>(
+ $"/api/v1/control/provisioning/jobs/{job.Id}/plan", cancellationToken);
+ Assert.Equal(expectedPackages, storedPlan!.Plan.Packages);
+ var confirmResponse = await client.PostAsJsonAsync(
+ $"/api/v1/control/provisioning/jobs/{job.Id}/confirm",
+ new { reason = "Reviewed generated plan", idempotencyKey = Guid.NewGuid().ToString() },
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, confirmResponse.StatusCode);
+ var confirmed = await confirmResponse.Content.ReadFromJsonAsync<
+ ServerMonitorManager.Core.ProvisioningJob>(cancellationToken);
+ Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, confirmed!.State);
+ Assert.Equal("confirmed-queued", confirmed.CurrentStep);
+ var grantRequest = new { idempotencyKey = Guid.NewGuid().ToString() };
+ var grantResponse = await agent.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant",
+ grantRequest,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, grantResponse.StatusCode);
+ var grant = await grantResponse.Content.ReadFromJsonAsync<
+ ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken);
+ var grantReplayResponse = await agent.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant",
+ grantRequest,
+ cancellationToken);
+ var grantReplay = await grantReplayResponse.Content.ReadFromJsonAsync<
+ ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken);
+ Assert.Equal(grant!.Signature, grantReplay!.Signature);
+ var authority = _factory.Services.GetRequiredService<
+ ServerMonitorManager.Control.CertificateAuthority>();
+ Assert.True(ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.Verify(
+ grant,
+ authority.PublicCertificate,
+ job.Id,
+ "home",
+ storedPlan.Plan,
+ DateTimeOffset.UtcNow));
+ Assert.Equal(HttpStatusCode.NoContent, (await agent.GetAsync(
+ "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);
+ Assert.Equal(HttpStatusCode.BadRequest, (await client.PostAsJsonAsync(
+ "/api/v1/control/agents/home/provisioning/jobs",
+ new
+ {
+ actionType = "system.base-install",
+ schemaVersion = 1,
+ parameters = new
+ {
+ timezone = "UTC",
+ locale = "en_US.UTF-8",
+ aptUpdate = true,
+ aptUpgrade = false,
+ packageCatalogVersion = 1,
+ packageGroupIds = new[] { "curl" },
+ swapMode = "disabled",
+ swapSizeMiB = (int?)null,
+ vmSwappiness = 60,
+ enableUnattendedUpgrades = false,
+ rebootPolicy = "never"
+ },
+ ttlMinutes = 60,
+ auditReason = "Reject arbitrary package input",
+ idempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken)).StatusCode);
+ Assert.Equal(HttpStatusCode.OK, (await client.GetAsync(
+ $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode);
+ using var eventsResponse = await client.GetAsync(
+ $"/api/v1/control/provisioning/jobs/{job.Id}/events?limit=10", cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, eventsResponse.StatusCode);
+ var events = await eventsResponse.Content.ReadFromJsonAsync(
+ cancellationToken);
+ Assert.NotNull(events);
+ Assert.NotEmpty(events);
+ }
+
+ [Fact]
+ public async Task AgentReceivesOnlyItsOwnProvisioningJob()
+ {
+ var nodeId = $"node-{Guid.NewGuid():N}"[..13];
+ var store = _factory.Services.GetRequiredService();
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var token = await store.CreateEnrollmentTokenAsync(nodeId, TimeSpan.FromMinutes(10), cancellationToken);
+ await store.EnrollAsync(
+ new ServerMonitorManager.Core.EnrollmentRequest(
+ nodeId, token, "csr", Guid.NewGuid().ToString()),
+ () => new ServerMonitorManager.Control.IssuedCertificate(
+ "certificate", "ca", Guid.NewGuid().ToString("N"), DateTimeOffset.UtcNow.AddDays(1)),
+ cancellationToken);
+ using var parameters = System.Text.Json.JsonDocument.Parse("{}");
+ var created = await store.CreateProvisioningJobAsync(
+ nodeId,
+ new ServerMonitorManager.Core.ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 60,
+ "API Agent isolation test", Guid.NewGuid().ToString()),
+ "windows-pc",
+ cancellationToken);
+
+ using var other = _factory.CreateClient();
+ other.DefaultRequestHeaders.Add("X-Test-Identity", "other-node");
+ other.DefaultRequestHeaders.Add("X-Test-Role", "Agent");
+ Assert.Equal(HttpStatusCode.NoContent, (await other.GetAsync(
+ "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);
+
+ using var assigned = _factory.CreateClient();
+ assigned.DefaultRequestHeaders.Add("X-Test-Identity", nodeId);
+ assigned.DefaultRequestHeaders.Add("X-Test-Role", "Agent");
+ var response = await assigned.GetAsync(
+ "/api/v1/agents/provisioning/jobs/next", cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ var claimed = await response.Content.ReadFromJsonAsync(
+ cancellationToken);
+ Assert.NotNull(claimed);
+ Assert.Equal(created.Id, claimed.Id);
+ Assert.Equal(nodeId, claimed.NodeId);
+ Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State);
+ Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync(
+ "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);
+
+ var preflightReport = new
+ {
+ facts = new
+ {
+ operatingSystem = "ubuntu",
+ operatingSystemVersion = "24.04",
+ architecture = "x64",
+ hasSystemd = true,
+ hasSshd = true,
+ hasNftables = true,
+ hasWireGuard = false,
+ hasApt = true
+ },
+ observedAt = DateTimeOffset.UtcNow,
+ idempotencyKey = Guid.NewGuid().ToString()
+ };
+ Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
+ preflightReport,
+ cancellationToken)).StatusCode);
+ using var factsResponse = await assigned.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
+ preflightReport,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, factsResponse.StatusCode);
+ var recorded = await factsResponse.Content.ReadFromJsonAsync(
+ cancellationToken);
+ Assert.Equal(nodeId, recorded!.NodeId);
+ Assert.Equal("ubuntu", recorded.Facts.OperatingSystem);
+ using var replayResponse = await assigned.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
+ preflightReport,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, replayResponse.StatusCode);
+ var replayed = await replayResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal(recorded.UpdatedAt, replayed!.UpdatedAt);
+
+ using var operatorClient = _factory.CreateClient();
+ operatorClient.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc");
+ operatorClient.DefaultRequestHeaders.Add("X-Test-Role", "Operator");
+ using var storedFactsResponse = await operatorClient.GetAsync(
+ $"/api/v1/control/agents/{nodeId}/facts/preflight", cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, storedFactsResponse.StatusCode);
+ var stored = await storedFactsResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal(created.Id, stored!.SourceJobId);
+
+ var desiredRequest = new
+ {
+ schemaVersion = 1,
+ desired = new
+ {
+ requireSystemd = true,
+ requireSshd = true,
+ requireNftables = true,
+ requireWireGuard = true,
+ requireApt = true,
+ allowedArchitectures = new[] { "x64", "arm64" }
+ },
+ auditReason = "Detect missing host capabilities",
+ idempotencyKey = Guid.NewGuid().ToString()
+ };
+ using var desiredResponse = await operatorClient.PutAsJsonAsync(
+ $"/api/v1/control/agents/{nodeId}/desired/preflight",
+ desiredRequest,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, desiredResponse.StatusCode);
+ var desired = await desiredResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal(1, desired!.Version);
+ using var desiredReplayResponse = await operatorClient.PutAsJsonAsync(
+ $"/api/v1/control/agents/{nodeId}/desired/preflight",
+ desiredRequest,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, desiredReplayResponse.StatusCode);
+ var desiredReplay = await desiredReplayResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal(desired.UpdatedAt, desiredReplay!.UpdatedAt);
+
+ using var driftResponse = await operatorClient.GetAsync(
+ $"/api/v1/control/agents/{nodeId}/drift/preflight", cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, driftResponse.StatusCode);
+ var drift = await driftResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal("Drifted", drift!.Status);
+ Assert.Equal(["wireguard.missing"], drift.DriftCodes);
+
+ var synchronizedDesiredRequest = new
+ {
+ schemaVersion = 1,
+ desired = new
+ {
+ requireSystemd = true,
+ requireSshd = true,
+ requireNftables = true,
+ requireWireGuard = false,
+ requireApt = true,
+ allowedArchitectures = new[] { "x64", "arm64" }
+ },
+ auditReason = "Accept host without WireGuard tooling",
+ idempotencyKey = Guid.NewGuid().ToString()
+ };
+ using var synchronizedDesiredResponse = await operatorClient.PutAsJsonAsync(
+ $"/api/v1/control/agents/{nodeId}/desired/preflight",
+ synchronizedDesiredRequest,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, synchronizedDesiredResponse.StatusCode);
+ var synchronizedDesired = await synchronizedDesiredResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal(2, synchronizedDesired!.Version);
+ using var synchronizedDriftResponse = await operatorClient.GetAsync(
+ $"/api/v1/control/agents/{nodeId}/drift/preflight", cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, synchronizedDriftResponse.StatusCode);
+ var synchronizedDrift = await synchronizedDriftResponse.Content
+ .ReadFromJsonAsync(cancellationToken);
+ Assert.Equal("InSync", synchronizedDrift!.Status);
+ Assert.Empty(synchronizedDrift.DriftCodes);
+
+ var progress = new
+ {
+ state = "Running",
+ progressPercent = 25,
+ step = "apply",
+ eventCode = "apply.started",
+ message = "Applying the approved plan.",
+ idempotencyKey = Guid.NewGuid().ToString()
+ };
+ Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{created.Id}/progress",
+ progress,
+ cancellationToken)).StatusCode);
+ using var progressResponse = await assigned.PostAsJsonAsync(
+ $"/api/v1/agents/provisioning/jobs/{created.Id}/progress",
+ progress,
+ cancellationToken);
+ Assert.Equal(HttpStatusCode.OK, progressResponse.StatusCode);
+ var updated = await progressResponse.Content.ReadFromJsonAsync(
+ cancellationToken);
+ Assert.Equal(25, updated!.ProgressPercent);
+ Assert.Equal("apply", updated.CurrentStep);
+ }
+
public async ValueTask DisposeAsync() => await _factory.DisposeAsync();
private sealed class ControlApiFactory : WebApplicationFactory
diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs
index 4c2757b..45eba97 100644
--- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs
@@ -1,6 +1,7 @@
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
+using System.Text.Json;
using ServerMonitorManager.Control;
using ServerMonitorManager.Core;
using Xunit;
@@ -105,7 +106,55 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
await verify.OpenAsync(cancellationToken);
var version = verify.CreateCommand();
version.CommandText = "PRAGMA user_version;";
- Assert.Equal(1L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
+ Assert.Equal(8L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
+ }
+
+ [Fact]
+ public async Task ExpiredProvisioningJobsAreCancelledOrRequireReconciliationAndCanRetry()
+ {
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var (store, _) = CreateServices();
+ await store.InitializeAsync(cancellationToken);
+ await EnrollAgentAsync(store, "queued-node", "AABB", cancellationToken);
+ await EnrollAgentAsync(store, "running-node", "CCDD", cancellationToken);
+ using var parameters = JsonDocument.Parse("{}");
+ var queued = await store.CreateProvisioningJobAsync(
+ "queued-node",
+ new ProvisioningJobCreateRequest(
+ "system.base-install", 1, parameters.RootElement.Clone(), 5,
+ "Await approval", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ var running = await store.CreateProvisioningJobAsync(
+ "running-node",
+ new ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 5,
+ "Inspect node", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken));
+
+ var result = await store.MaintainAsync(
+ running.ExpiresAt.AddSeconds(1), cancellationToken);
+
+ Assert.Equal(1, result.ProvisioningJobsCancelled);
+ Assert.Equal(1, result.ProvisioningJobsNeedingReconciliation);
+ Assert.Equal(ProvisioningJobStates.Cancelled,
+ (await store.GetProvisioningJobAsync(queued.Id, cancellationToken))!.State);
+ var reconciliation = await store.GetProvisioningJobAsync(running.Id, cancellationToken);
+ Assert.Equal(ProvisioningJobStates.NeedsReconciliation, reconciliation!.State);
+ Assert.Equal("job.ttl_expired", reconciliation.LastError);
+
+ var retried = await store.RetryProvisioningJobAsync(
+ running.Id,
+ new ProvisioningJobCommandRequest("Factual state checked", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Queued, retried!.State);
+ Assert.Equal(0, retried.ProgressPercent);
+ Assert.Null(retried.LastError);
+ Assert.True(retried.ExpiresAt > running.ExpiresAt);
+ Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken));
}
[Fact]
diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs
index beb3c61..d17e00e 100644
--- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs
@@ -1,5 +1,6 @@
using Microsoft.Extensions.Options;
using Microsoft.Data.Sqlite;
+using System.Text.Json;
using ServerMonitorManager.Control;
using ServerMonitorManager.Core;
using Xunit;
@@ -10,6 +11,293 @@ public sealed class ControlStoreTests : IAsyncDisposable
{
private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-tests-{Guid.NewGuid():N}");
+ [Fact]
+ public async Task VersionOneDatabaseMigratesToProvisioningSchema()
+ {
+ Directory.CreateDirectory(_directory);
+ var databasePath = Path.Combine(_directory, "control.db");
+ await using (var connection = new SqliteConnection($"Data Source={databasePath}"))
+ {
+ await connection.OpenAsync(TestContext.Current.CancellationToken);
+ var command = connection.CreateCommand();
+ command.CommandText = "PRAGMA user_version = 1;";
+ await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken);
+ }
+
+ var store = CreateStore();
+ await store.InitializeAsync(TestContext.Current.CancellationToken);
+
+ await using var migrated = new SqliteConnection($"Data Source={databasePath}");
+ await migrated.OpenAsync(TestContext.Current.CancellationToken);
+ var version = migrated.CreateCommand();
+ version.CommandText = "PRAGMA user_version;";
+ Assert.Equal(8L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
+ var table = migrated.CreateCommand();
+ table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');";
+ Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
+ }
+
+ [Fact]
+ public async Task ProvisioningJobRequiresConfirmationIsIdempotentAndLocksNode()
+ {
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var store = CreateStore();
+ await store.InitializeAsync(cancellationToken);
+ await EnrollAgentAsync(store, "home", "A1B2", cancellationToken);
+ var parameters = CreateBaseInstallParameters();
+ var request = new ProvisioningJobCreateRequest(
+ "system.base-install", 1, parameters, 60,
+ "Prepare test server", Guid.NewGuid().ToString());
+
+ var created = await store.CreateProvisioningJobAsync("home", request, "operator", cancellationToken);
+ var replay = await store.CreateProvisioningJobAsync("home", request, "operator", cancellationToken);
+
+ Assert.Equal(created.Id, replay.Id);
+ Assert.Equal(created.State, replay.State);
+ Assert.Equal(ProvisioningJobStates.Queued, created.State);
+ Assert.True(created.ConfirmationRequired);
+ await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync(
+ "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken));
+
+ var confirmation = new ProvisioningJobCommandRequest(
+ "Approved for test", Guid.NewGuid().ToString());
+ await Assert.ThrowsAsync(() => store.ConfirmProvisioningJobAsync(
+ created.Id, confirmation, "operator", cancellationToken));
+
+ var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Preflight, claimed!.State);
+ var expectedPlan = CreateBaseInstallPlan();
+ var invalidPlan = expectedPlan with { Packages = [.. expectedPlan.Packages, "untrusted-package"] };
+ await Assert.ThrowsAsync(() => store.RecordBaseInstallPlanAsync(
+ "home", created.Id,
+ new SystemBaseInstallPlanReportRequest(invalidPlan, Guid.NewGuid().ToString()),
+ cancellationToken));
+
+ var planRequest = new SystemBaseInstallPlanReportRequest(
+ expectedPlan, Guid.NewGuid().ToString());
+ var recorded = await store.RecordBaseInstallPlanAsync(
+ "home", created.Id, planRequest, cancellationToken);
+ var planReplay = await store.RecordBaseInstallPlanAsync(
+ "home", created.Id, planRequest, cancellationToken);
+ Assert.Equal(recorded!.JobId, planReplay!.JobId);
+ Assert.Equal(recorded.NodeId, planReplay.NodeId);
+ Assert.Equal(recorded.CreatedAt, planReplay.CreatedAt);
+ Assert.Equal(recorded.Plan.Packages, planReplay.Plan.Packages);
+ Assert.Equal(expectedPlan.Packages, recorded!.Plan.Packages);
+ Assert.Equal(
+ ProvisioningJobStates.AwaitingConfirmation,
+ (await store.GetProvisioningJobAsync(created.Id, cancellationToken))!.State);
+ var readablePlan = await store.GetBaseInstallPlanAsync(created.Id, cancellationToken);
+ Assert.Equal(expectedPlan.Packages, readablePlan!.Plan.Packages);
+
+ var confirmed = await store.ConfirmProvisioningJobAsync(
+ created.Id, confirmation, "operator", cancellationToken);
+ var confirmationReplay = await store.ConfirmProvisioningJobAsync(
+ created.Id, confirmation, "operator", cancellationToken);
+ Assert.Equal(confirmed!.Id, confirmationReplay!.Id);
+ Assert.Equal(confirmed.State, confirmationReplay.State);
+ Assert.Equal(ProvisioningJobStates.Queued, confirmed.State);
+ Assert.Equal("confirmed-queued", confirmed.CurrentStep);
+ Assert.NotNull(confirmed.ConfirmedAt);
+ Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+
+ var cancelled = await store.CancelProvisioningJobAsync(
+ created.Id,
+ new ProvisioningJobCommandRequest("Test completed", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State);
+ Assert.NotNull(cancelled.CancelledAt);
+
+ var next = await store.CreateProvisioningJobAsync(
+ "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Queued, next.State);
+ }
+
+ [Fact]
+ public async Task ProvisioningJobCanBeClaimedOnlyOnceByAssignedNode()
+ {
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var store = CreateStore();
+ await store.InitializeAsync(cancellationToken);
+ await EnrollAgentAsync(store, "home", "C3D4", cancellationToken);
+ await EnrollAgentAsync(store, "other", "E5F6", cancellationToken);
+ using var parameters = JsonDocument.Parse("{}");
+ var created = await store.CreateProvisioningJobAsync(
+ "home",
+ new ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 60,
+ "Inspect server", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+
+ Assert.Null(await store.ClaimNextProvisioningJobAsync("other", cancellationToken));
+ var claims = await Task.WhenAll(
+ store.ClaimNextProvisioningJobAsync("home", cancellationToken),
+ store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+
+ var claimed = Assert.Single(claims, job => job is not null)!;
+ Assert.Equal(created.Id, claimed.Id);
+ Assert.Equal("home", claimed.NodeId);
+ Assert.Equal(ProvisioningJobStates.Preflight, claimed.State);
+ Assert.Null(Assert.Single(claims, job => job is null));
+
+ var preflight = new ProvisioningJobProgressRequest(
+ ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress",
+ "secret-token-should-never-be-persisted", Guid.NewGuid().ToString());
+ Assert.Null(await store.ReportProvisioningProgressAsync(
+ "other", created.Id, preflight, cancellationToken));
+ var firstProgress = await store.ReportProvisioningProgressAsync(
+ "home", created.Id, preflight, cancellationToken);
+ var replay = await store.ReportProvisioningProgressAsync(
+ "home", created.Id, preflight, cancellationToken);
+ Assert.Equal(firstProgress!.Id, replay!.Id);
+ Assert.Equal(10, replay.ProgressPercent);
+ Assert.Equal("inspect-os", replay.CurrentStep);
+ await Assert.ThrowsAsync(() =>
+ store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ preflight with
+ {
+ State = ProvisioningJobStates.Completed,
+ ProgressPercent = 100,
+ IdempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken));
+
+ var running = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ preflight with
+ {
+ State = ProvisioningJobStates.Running,
+ ProgressPercent = 40,
+ Step = "apply",
+ EventCode = "apply.started",
+ IdempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken);
+ var verifying = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ preflight with
+ {
+ State = ProvisioningJobStates.Verifying,
+ ProgressPercent = 90,
+ Step = "verify",
+ EventCode = "verify.started",
+ IdempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken);
+ var completed = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ preflight with
+ {
+ State = ProvisioningJobStates.Completed,
+ ProgressPercent = 100,
+ Step = "complete",
+ EventCode = "job.completed",
+ IdempotencyKey = Guid.NewGuid().ToString()
+ },
+ cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Running, running!.State);
+ Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State);
+ Assert.Equal(ProvisioningJobStates.Completed, completed!.State);
+ var events = await store.ListProvisioningEventsAsync(created.Id, 100, cancellationToken);
+ Assert.NotNull(events);
+ var preflightEvent = Assert.Single(events!, item => item.EventType == "preflight.progress");
+ Assert.Equal("inspect-os", preflightEvent.Step);
+ Assert.Equal(10, preflightEvent.ProgressPercent);
+ Assert.DoesNotContain(
+ events!, item => item.Message.Contains("secret-token", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public async Task FailedJobRollbackIsNodeScopedRecoverableAndTerminal()
+ {
+ var cancellationToken = TestContext.Current.CancellationToken;
+ var store = CreateStore();
+ await store.InitializeAsync(cancellationToken);
+ await EnrollAgentAsync(store, "home", "7788", cancellationToken);
+ using var parameters = JsonDocument.Parse("{}");
+ var created = await store.CreateProvisioningJobAsync(
+ "home",
+ new ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 5,
+ "Rollback test", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+ var failed = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ new ProvisioningJobProgressRequest(
+ ProvisioningJobStates.Failed, 20, "inspect-os", "preflight.failed",
+ "Preflight failed.", Guid.NewGuid().ToString()),
+ cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Failed, failed!.State);
+ await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync(
+ "home",
+ new ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 5,
+ "Must remain blocked", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken));
+
+ var rollbackRequest = new ProvisioningJobCommandRequest(
+ "Restore preflight state", Guid.NewGuid().ToString());
+ var queued = await store.StartProvisioningRollbackAsync(
+ created.Id, rollbackRequest, "operator", cancellationToken);
+ var replay = await store.StartProvisioningRollbackAsync(
+ created.Id, rollbackRequest, "operator", cancellationToken);
+ Assert.Equal(queued!.Id, replay!.Id);
+ Assert.Equal("rollback-queued", queued.CurrentStep);
+ var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken);
+ Assert.Equal(ProvisioningJobStates.RollingBack, claimed!.State);
+ Assert.Equal("rollback", claimed.CurrentStep);
+ Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+
+ var maintenance = await store.MaintainAsync(
+ queued.ExpiresAt.AddSeconds(1), cancellationToken);
+ Assert.Equal(1, maintenance.ProvisioningJobsNeedingReconciliation);
+ var uncertain = await store.GetProvisioningJobAsync(created.Id, cancellationToken);
+ Assert.Equal(ProvisioningJobStates.NeedsReconciliation, uncertain!.State);
+ Assert.Equal("rollback-reconcile", uncertain.CurrentStep);
+ await Assert.ThrowsAsync(() =>
+ store.RetryProvisioningJobAsync(
+ created.Id,
+ new ProvisioningJobCommandRequest("Unsafe retry", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken));
+
+ await store.StartProvisioningRollbackAsync(
+ created.Id,
+ new ProvisioningJobCommandRequest("Resume rollback", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+ var rolling = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ new ProvisioningJobProgressRequest(
+ ProvisioningJobStates.RollingBack, 50, "restore", "rollback.progress",
+ "Restoring backup.", Guid.NewGuid().ToString()),
+ cancellationToken);
+ var rolledBack = await store.ReportProvisioningProgressAsync(
+ "home", created.Id,
+ new ProvisioningJobProgressRequest(
+ ProvisioningJobStates.RolledBack, 100, "rollback-complete", "rollback.completed",
+ "Backup restored.", Guid.NewGuid().ToString()),
+ cancellationToken);
+ Assert.Equal(50, rolling!.ProgressPercent);
+ Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State);
+ Assert.Null(rolledBack.LastError);
+ var replacement = await store.CreateProvisioningJobAsync(
+ "home",
+ new ProvisioningJobCreateRequest(
+ "preflight", 1, parameters.RootElement.Clone(), 5,
+ "Allowed after rollback", Guid.NewGuid().ToString()),
+ "operator",
+ cancellationToken);
+ Assert.Equal(ProvisioningJobStates.Queued, replacement.State);
+ }
+
[Fact]
public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates()
{
@@ -487,6 +775,19 @@ public sealed class ControlStoreTests : IAsyncDisposable
}));
}
+ private static JsonElement CreateBaseInstallParameters()
+ => JsonSerializer.SerializeToElement(
+ new SystemBaseInstallParameters(
+ "UTC", "en_US.UTF-8", true, false, 1, ["core"],
+ "disabled", null, 60, true, "never"),
+ SmmJsonContext.Default.SystemBaseInstallParameters);
+
+ private static SystemBaseInstallPlan CreateBaseInstallPlan()
+ => new(
+ "UTC", "en_US.UTF-8", true, false,
+ SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]),
+ "disabled", null, 60, true, "never", []);
+
private static async Task EnrollAgentAsync(
ControlStore store,
string nodeId,
diff --git a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs
new file mode 100644
index 0000000..2ed11c2
--- /dev/null
+++ b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs
@@ -0,0 +1,91 @@
+using System.Text.Json;
+using ServerMonitorManager.Core;
+using ServerMonitorManager.Provisioning.Helper;
+using Xunit;
+
+namespace ServerMonitorManager.Control.Tests;
+
+public sealed class ProvisioningHelperTests
+{
+ [Fact]
+ public void HelperRejectsEveryActionOutsideFixedAllowlist()
+ {
+ using var document = JsonDocument.Parse("{}");
+ var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest(
+ "1", new string('a', 32), "shell", 1,
+ ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone()));
+
+ Assert.False(response.Success);
+ Assert.Equal("action.denied", response.Code);
+ Assert.Null(response.Preflight);
+ }
+
+ [Fact]
+ public void HelperAcceptsOnlyEmptyPreflightSchemaOne()
+ {
+ if (!OperatingSystem.IsLinux())
+ {
+ return;
+ }
+
+ using var document = JsonDocument.Parse("{}");
+ var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest(
+ "1", new string('b', 32), "preflight", 1,
+ ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone()));
+
+ Assert.True(response.Success);
+ Assert.Equal("preflight.completed", response.Code);
+ Assert.NotNull(response.Preflight);
+ Assert.NotEmpty(response.Preflight.OperatingSystem);
+ Assert.NotEmpty(response.Preflight.Architecture);
+ }
+
+ [Fact]
+ public void HelperContractRejectsUnknownJsonMembers()
+ {
+ const string json = """
+ {"protocolVersion":"1","jobId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "actionType":"preflight","schemaVersion":1,
+ "moduleHash":"2dc48fb4528a291221954fc2dd3478d431b66fe34228f29684ce1648dbe2f32b",
+ "parameters":{},"command":"id"}
+ """;
+
+ Assert.Throws(() =>
+ JsonSerializer.Deserialize(json, SmmJsonContext.Default.ProvisioningHelperRequest));
+ }
+
+ [Fact]
+ public void BaseInstallSchemaRejectsCommandText()
+ {
+ const string json = """
+ {"timezone":"UTC","locale":"en_US.UTF-8","aptUpdate":true,"aptUpgrade":false,
+ "packageCatalogVersion":1,"packageGroupIds":["core"],"swapMode":"disabled",
+ "swapSizeMiB":null,"vmSwappiness":60,"enableUnattendedUpgrades":true,
+ "rebootPolicy":"never","command":"id"}
+ """;
+
+ Assert.Throws(() =>
+ JsonSerializer.Deserialize(json, SmmJsonContext.Default.SystemBaseInstallParameters));
+ }
+
+ [Fact]
+ public void HelperBuildsDeterministicBaseInstallPlanWithoutCommands()
+ {
+ var parameters = new SystemBaseInstallParameters(
+ "UTC", "en_US.UTF-8", true, false, 1,
+ ["development", "core"], "disabled", null, 60, true, "never");
+ var json = JsonSerializer.SerializeToElement(
+ parameters, SmmJsonContext.Default.SystemBaseInstallParameters);
+ var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest(
+ "1", new string('c', 32), "system.base-install", 1,
+ ProvisioningActionCatalog.SystemBaseInstallModuleHash, json));
+
+ Assert.True(response.Success);
+ Assert.Equal("system.base-install.plan-ready", response.Code);
+ Assert.Null(response.Preflight);
+ Assert.Equal(
+ ["ca-certificates", "curl", "jq", "build-essential", "git"],
+ response.BaseInstallPlan!.Packages);
+ Assert.Equal("never", response.BaseInstallPlan.RebootPolicy);
+ }
+}
diff --git a/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj b/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj
index 9e9bb6b..5413588 100644
--- a/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj
+++ b/tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj
@@ -8,6 +8,7 @@
+
diff --git a/tests/bootstrap/run-native-systemd-smoke.sh b/tests/bootstrap/run-native-systemd-smoke.sh
new file mode 100755
index 0000000..8789577
--- /dev/null
+++ b/tests/bootstrap/run-native-systemd-smoke.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+archive="${1:?usage: run-native-systemd-smoke.sh ARCHIVE BOOTSTRAP}"
+bootstrap="${2:?usage: run-native-systemd-smoke.sh ARCHIVE BOOTSTRAP}"
+port="${SMM_SMOKE_PORT:-17443}"
+
+cleanup() {
+ sudo "$bootstrap" uninstall-control --confirm-destroy-control >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+sudo "$bootstrap" preflight
+sudo "$bootstrap" verify-release "$archive"
+sudo "$bootstrap" install-control "$archive" 127.0.0.1 "$port"
+sudo test -x /usr/local/sbin/ochenstarik-smm-emergency
+sudo /usr/local/sbin/ochenstarik-smm-emergency status
+
+for _ in {1..30}; do
+ if sudo curl --fail --silent \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ "https://127.0.0.1:$port/healthz" >/dev/null; then
+ break
+ fi
+ sleep 1
+done
+sudo curl --fail --silent --show-error \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ "https://127.0.0.1:$port/healthz"
+
+sudo "$bootstrap" install-control "$archive" 127.0.0.1 "$port"
+sudo systemctl restart ochenstarik-smm-control.service
+sudo systemctl is-active --quiet ochenstarik-smm-control.service
+sudo curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ "https://127.0.0.1:$port/healthz"
+
+printf '%s\n' "NATIVE_SYSTEMD_SMOKE=PASS"
diff --git a/tests/bootstrap/run-systemd-container-smoke.sh b/tests/bootstrap/run-systemd-container-smoke.sh
new file mode 100755
index 0000000..42d25e2
--- /dev/null
+++ b/tests/bootstrap/run-systemd-container-smoke.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+base_image="${1:?usage: run-systemd-container-smoke.sh BASE_IMAGE ARCHIVE BOOTSTRAP}"
+archive="$(realpath "${2:?archive is required}")"
+bootstrap="$(realpath "${3:?bootstrap is required}")"
+name="smm-systemd-${RANDOM}-${RANDOM}"
+image="smm-systemd-smoke:${base_image//[:\/]/-}"
+port="17443"
+smoke_dir="/root/smm-smoke"
+remote_archive="$smoke_dir/release.tar.gz"
+remote_bootstrap="$smoke_dir/ochenstarik-server-monitor-manager.sh"
+
+cleanup() {
+ docker rm -f "$name" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+docker build \
+ --build-arg "BASE_IMAGE=$base_image" \
+ -f "$root/tests/bootstrap/systemd-container.Dockerfile" \
+ -t "$image" \
+ "$root"
+docker run --detach --privileged --cgroupns=private \
+ --tmpfs /run --tmpfs /run/lock --name "$name" "$image" >/dev/null
+
+for _ in {1..30}; do
+ if docker exec "$name" systemctl is-system-running >/dev/null 2>&1; then
+ break
+ fi
+ state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)"
+ [[ "$state" == "degraded" ]] && break
+ sleep 1
+done
+state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)"
+[[ "$state" == "running" || "$state" == "degraded" ]] || {
+ docker exec "$name" systemctl --failed --no-pager || true
+ printf '%s\n' "container systemd did not finish booting: $state" >&2
+ exit 1
+}
+
+docker exec "$name" install -d -m 0700 "$smoke_dir"
+docker cp "$archive" "$name:$remote_archive"
+docker cp "${archive}.sha256" "$name:${remote_archive}.sha256"
+docker cp "$bootstrap" "$name:$remote_bootstrap"
+docker exec "$name" chmod 0700 "$remote_bootstrap"
+docker exec "$name" "$remote_bootstrap" preflight
+docker exec "$name" "$remote_bootstrap" install-control \
+ "$remote_archive" 127.0.0.1 "$port"
+docker exec "$name" curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ "https://127.0.0.1:$port/healthz"
+docker exec "$name" "$remote_bootstrap" install-control \
+ "$remote_archive" 127.0.0.1 "$port"
+
+docker restart "$name" >/dev/null
+for _ in {1..60}; do
+ if docker exec "$name" systemctl is-active --quiet ochenstarik-smm-control.service; then
+ break
+ fi
+ sleep 1
+done
+docker exec "$name" systemctl is-active --quiet ochenstarik-smm-control.service
+docker exec "$name" curl --fail --silent --show-error --retry 15 --retry-all-errors --retry-delay 1 \
+ --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
+ "https://127.0.0.1:$port/healthz"
+docker exec "$name" /usr/local/sbin/ochenstarik-smm-emergency status
+
+printf '%s\n' "SYSTEMD_CONTAINER_SMOKE=PASS image=$base_image"
diff --git a/tests/bootstrap/systemd-container.Dockerfile b/tests/bootstrap/systemd-container.Dockerfile
new file mode 100644
index 0000000..3790429
--- /dev/null
+++ b/tests/bootstrap/systemd-container.Dockerfile
@@ -0,0 +1,14 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ENV container=docker
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ ca-certificates curl iproute2 openssl procps sudo systemd systemd-sysv \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]
diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh
new file mode 100755
index 0000000..135cdee
--- /dev/null
+++ b/tests/bootstrap/test-bootstrap-contract.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
+helper="$root/deploy/ochenstarik-smm-policy-apply"
+emergency="$root/deploy/ochenstarik-smm-emergency"
+
+help_output="$(bash "$bootstrap" --help)"
+version_output="$(bash "$bootstrap" --version)"
+
+grep -Fq "install-control ARCHIVE PUBLIC_HOST" <<<"$help_output"
+grep -Fq "install-agent ARCHIVE NODE_ID CONTROL_URL CA_CERT" <<<"$help_output"
+grep -Fq "install-node ARCHIVE" <<<"$help_output"
+grep -Fq "mesh-init PUBLIC_ENDPOINT" <<<"$help_output"
+grep -Fq "peer-add SMMPEER1_CODE" <<<"$help_output"
+grep -Fq "mesh-status" <<<"$help_output"
+grep -Fq "SMM_ENROLL_TOKEN" <<<"$help_output"
+grep -Fq "node-code NODE_ID" <<<"$help_output"
+grep -Fq "verify-release ARCHIVE" <<<"$help_output"
+grep -Fq "node-token NODE_ID" <<<"$help_output"
+grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output"
+emergency_help="$(bash "$emergency" --help)"
+grep -Fq 'mesh-disable' <<<"$emergency_help"
+grep -Fq 'firewall-restore' <<<"$emergency_help"
+
+if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then
+ printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2
+ exit 1
+fi
+
+if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>&1; then
+ printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2
+ exit 1
+fi
+if bash "$emergency" mesh-disable >/dev/null 2>&1; then
+ printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2
+ exit 1
+fi
+
+policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)"
+printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state"
+printf 'target\t10.77.0.3\tkey-target\tactive\n' >>"$policy_state"
+connect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
+ bash "$helper" link-connect source target tcp 22 10)"
+grep -Fq 'ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22' <<<"$connect_output"
+grep -Fq 'smm:source:target:tcp:22' <<<"$connect_output"
+disconnect_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
+ bash "$helper" link-disconnect source target tcp 22)"
+grep -Fq 'smm:source:target:tcp:22' <<<"$disconnect_output"
+rm -f -- "$policy_state"
+
+fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)"
+trap 'rm -rf -- "$fixture"' EXIT
+mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap"
+install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent"
+install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control"
+install -m 0755 /bin/true "$fixture/payload/provisioning-helper/ochenstarik-smm-provisioning-helper"
+install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply"
+install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency"
+install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/"
+install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/"
+install -m 0644 "$root/deploy/ochenstarik-smm-provisioning-helper.service" "$fixture/payload/deploy/"
+install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/"
+install -m 0755 "$bootstrap" "$fixture/payload/bootstrap/ochenstarik-server-monitor-manager.sh"
+tar -C "$fixture/payload" -czf "$fixture/release.tar.gz" agent control provisioning-helper deploy bootstrap
+sha256sum "$fixture/release.tar.gz" >"$fixture/release.tar.gz.sha256"
+bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null
+
+printf '%064d %s\n' 0 release.tar.gz >"$fixture/release.tar.gz.sha256"
+if bash "$bootstrap" verify-release "$fixture/release.tar.gz" >/dev/null 2>&1; then
+ printf '%s\n' "corrupt release checksum unexpectedly succeeded" >&2
+ exit 1
+fi
+
+printf '%s\n' "BOOTSTRAP_CONTRACT=PASS"