From 470034f1376026eda5a52802b4080feb352357ec Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 14:38:05 +0700 Subject: [PATCH] Add local Mesh emergency recovery --- .github/workflows/linux-control-agent.yml | 7 + .github/workflows/linux-release.yml | 3 + deploy/ochenstarik-server-monitor-manager.sh | 8 ++ deploy/ochenstarik-smm-emergency | 129 +++++++++++++++++++ docs/installer-contract.md | 8 +- docs/linux-bootstrap.md | 13 ++ docs/roadmap.md | 2 +- tests/bootstrap/test-bootstrap-contract.sh | 9 ++ 8 files changed, 175 insertions(+), 4 deletions(-) create mode 100755 deploy/ochenstarik-smm-emergency diff --git a/.github/workflows/linux-control-agent.yml b/.github/workflows/linux-control-agent.yml index 80617ef..709fa4b 100644 --- a/.github/workflows/linux-control-agent.yml +++ b/.github/workflows/linux-control-agent.yml @@ -41,8 +41,10 @@ jobs: run: | bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply + bash -n deploy/ochenstarik-smm-emergency shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-smm-emergency bash tests/bootstrap/test-bootstrap-contract.sh - name: Publish agent amd64 @@ -64,6 +66,7 @@ jobs: install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/ + install -m 0755 deploy/ochenstarik-smm-emergency smoke/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/ tar -C smoke -czf smoke-release.tar.gz agent control deploy bootstrap sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256 @@ -90,6 +93,10 @@ jobs: sudo deploy/ochenstarik-server-monitor-manager.sh install-control \ smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \ || report_failure 'Initial Control installation failed' + sudo test -x /usr/local/sbin/ochenstarik-smm-emergency \ + || report_failure 'Emergency recovery command was not installed' + sudo /usr/local/sbin/ochenstarik-smm-emergency status >>"$smoke_log" 2>&1 \ + || report_failure 'Emergency recovery status failed' for _ in {1..30}; do sudo curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \ https://127.0.0.1:7443/healthz && break diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index 5ab5706..c2f3e87 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -20,8 +20,10 @@ jobs: run: | bash -n deploy/ochenstarik-server-monitor-manager.sh bash -n deploy/ochenstarik-smm-policy-apply + bash -n deploy/ochenstarik-smm-emergency shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh shellcheck --severity=error deploy/ochenstarik-smm-policy-apply + shellcheck --severity=error deploy/ochenstarik-smm-emergency bash tests/bootstrap/test-bootstrap-contract.sh - name: Package bootstrap @@ -87,6 +89,7 @@ jobs: install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/ install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/ install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/ + install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/ install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/ archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz" tar -C out -czf "$archive" agent control deploy bootstrap diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 66a7334..336acc7 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -13,6 +13,7 @@ readonly AGENT_USER="ochenstarik-smm-agent" readonly CONTROL_UNIT="ochenstarik-smm-control.service" readonly AGENT_UNIT="ochenstarik-smm-agent.service" readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply" +readonly EMERGENCY_COMMAND="/usr/local/sbin/ochenstarik-smm-emergency" readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control" readonly MESH_DIR="${STATE_DIR}/mesh" readonly WG_DIR="${ETC_DIR}/wireguard" @@ -152,6 +153,8 @@ extract_archive() { [[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing from archive." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency command is missing from archive." } verify_release_payload() { @@ -162,6 +165,7 @@ verify_release_payload() { [[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing." [[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing." [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing." + [[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency recovery command is missing." [[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing." [[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing." log "Release archive and checksum are valid." @@ -483,6 +487,8 @@ EOF chmod 0644 "$ETC_DIR/control-public-url" install -d -m 0755 "$(dirname "$POLICY_HELPER")" install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER" + install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" + install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND" install -d -m 0755 "$LIB_DIR/bootstrap" install -m 0644 "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$LIB_DIR/bootstrap/$FIREWALL_UNIT" printf '%s\n' "$CONTROL_USER ALL=(root) NOPASSWD: $POLICY_HELPER *" >"$SUDOERS_FILE" @@ -530,6 +536,8 @@ install_agent() { install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR" install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent" install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root" + install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" + install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND" if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then install -m 0600 -o "$AGENT_USER" -g "$AGENT_USER" "$ca_cert" "$ETC_DIR/control-ca.crt" else diff --git a/deploy/ochenstarik-smm-emergency b/deploy/ochenstarik-smm-emergency new file mode 100755 index 0000000..51cbad9 --- /dev/null +++ b/deploy/ochenstarik-smm-emergency @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +IFS=$'\n\t' + +readonly PROGRAM="ochenstarik-smm-emergency" +readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager" +readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager" +readonly MARKER="$STATE_DIR/mesh/emergency-disabled" +readonly CONTROL_UNIT="ochenstarik-smm-control.service" +readonly AGENT_UNIT="ochenstarik-smm-agent.service" +readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service" +readonly WIREGUARD_UNIT="wg-quick@smm0.service" +readonly NFT_FAMILY="inet" +readonly NFT_TABLE="ochenstarik_smm" + +fail() { printf '%s: %s\n' "$PROGRAM" "$*" >&2; exit 78; } +log() { printf '%s: %s\n' "$PROGRAM" "$*"; } + +usage() { + cat <<'EOF' +Local Server Monitor Manager emergency recovery + +Usage: + ochenstarik-smm-emergency status + ochenstarik-smm-emergency mesh-disable + ochenstarik-smm-emergency mesh-enable + ochenstarik-smm-emergency firewall-restore + +Commands use only Server Monitor Manager-owned units, interface and nftables +table. They do not modify the host SSH service or unrelated firewall rules. +EOF +} + +require_root() { + [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "this action must run as root (use sudo)" +} + +unit_state() { + local unit="$1" + if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then + systemctl is-active "$unit" 2>/dev/null || true + else + printf '%s\n' "not-installed" + fi +} + +show_status() { + local unit + for unit in "$CONTROL_UNIT" "$AGENT_UNIT" "$FIREWALL_UNIT" "$WIREGUARD_UNIT"; do + printf '%s: %s\n' "$unit" "$(unit_state "$unit")" + done + if command -v ip >/dev/null 2>&1 && ip link show smm0 >/dev/null 2>&1; then + printf '%s\n' "mesh-interface: present" + else + printf '%s\n' "mesh-interface: absent" + fi + if command -v nft >/dev/null 2>&1 && nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then + printf '%s\n' "mesh-firewall: loaded" + else + printf '%s\n' "mesh-firewall: absent" + fi + [[ -f "$MARKER" ]] && printf '%s\n' "emergency-lock: active" || printf '%s\n' "emergency-lock: inactive" + printf '%s\n' "backups:" + find "$STATE_DIR/bootstrap-backups" -maxdepth 1 -type f \ + \( -name '*.tar.gz' -o -name '*.empty' \) -printf ' %f\n' 2>/dev/null | sort -r | head -n 10 || true +} + +delete_project_firewall() { + if command -v nft >/dev/null 2>&1; then + nft delete table "$NFT_FAMILY" "$NFT_TABLE" 2>/dev/null || true + fi +} + +mesh_disable() { + require_root + systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true + systemctl disable --now "$FIREWALL_UNIT" 2>/dev/null || true + delete_project_firewall + install -d -m 0700 "$(dirname "$MARKER")" + printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$MARKER" + chmod 0600 "$MARKER" + log "Mesh disabled locally; Control and SSH were not changed." +} + +restore_project_firewall() { + require_root + [[ -f "$ETC_DIR/mesh.nft" ]] || fail "managed firewall configuration is missing" + command -v nft >/dev/null 2>&1 || fail "nft is unavailable" + if ! nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then + nft --check -f "$ETC_DIR/mesh.nft" + fi + delete_project_firewall + if ! nft -f "$ETC_DIR/mesh.nft"; then + systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true + fail "managed firewall restore failed; Mesh was disabled to fail closed" + fi + if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then + systemctl enable "$FIREWALL_UNIT" >/dev/null + fi + log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links." +} + +mesh_enable() { + require_root + [[ -f /etc/wireguard/smm0.conf ]] || fail "WireGuard smm0 configuration is missing" + if [[ -f "$ETC_DIR/mesh.nft" ]]; then + restore_project_firewall + fi + systemctl enable "$WIREGUARD_UNIT" >/dev/null + systemctl restart "$WIREGUARD_UNIT" + rm -f -- "$MARKER" + log "Mesh enabled locally." +} + +main() { + local action="${1:-help}" + shift || true + [[ $# -eq 0 ]] || fail "unexpected arguments" + case "$action" in + help|-h|--help) usage ;; + status) show_status ;; + mesh-disable) mesh_disable ;; + mesh-enable) mesh_enable ;; + firewall-restore) restore_project_firewall ;; + *) fail "unknown action: $action" ;; + esac +} + +main "$@" diff --git a/docs/installer-contract.md b/docs/installer-contract.md index 29dfb3a..88981a5 100644 --- a/docs/installer-contract.md +++ b/docs/installer-contract.md @@ -82,11 +82,13 @@ control device-code DEVICE_ID control node-code NODE_ID control automation-token AUTOMATION_ID SOURCE_NODE_ID emergency status -emergency vpn-disable -emergency ssh-restore BACKUP_ID -emergency firewall-restore BACKUP_ID +emergency mesh-disable +emergency mesh-enable +emergency firewall-restore ``` +Текущая emergency-команда покрывает уже реализованные Mesh/WireGuard и project-owned nftables resources. Действия `vpn-disable` и `ssh-restore BACKUP_ID` добавляются вместе с соответствующими provisioning-модулями и их root-only backup format; до появления этих модулей команда намеренно их отклоняет. + CLI является non-interactive, кроме локального ввода enrollment code и явных подтверждений опасного удаления. Машиночитаемый режим возвращает versioned JSON и стабильные exit codes. ## 6. Идемпотентность и обновление diff --git a/docs/linux-bootstrap.md b/docs/linux-bootstrap.md index 9e32cbe..13e73c3 100644 --- a/docs/linux-bootstrap.md +++ b/docs/linux-bootstrap.md @@ -96,3 +96,16 @@ sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy ``` Update создаёт root-only backup перед заменой binaries и автоматически восстанавливает предыдущую версию, если сервис не запускается. Перед alpha-тестом на реальных серверах обязательно сохраните отдельную консольную/SSH-сессию и не закрывайте основной административный доступ firewall-правилами проекта. + +## Локальное аварийное восстановление + +Установщик размещает независимую от Control Hub команду `/usr/local/sbin/ochenstarik-smm-emergency`. Она принимает только фиксированные действия и управляет исключительно интерфейсом `smm0`, systemd units и nftables-таблицей Server Monitor Manager: + +```bash +sudo ochenstarik-smm-emergency status +sudo ochenstarik-smm-emergency mesh-disable +sudo ochenstarik-smm-emergency firewall-restore +sudo ochenstarik-smm-emergency mesh-enable +``` + +`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0665d4a..d661f84 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -100,7 +100,7 @@ - [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов; - [x] реализовать nftables policy helper вместо временного deny-by-default helper; - [ ] добавить VM CI matrix, повторную установку и reboot; -- [ ] добавить локальную emergency recovery command. +- [x] добавить локальную emergency recovery command для текущих Mesh/firewall-компонентов. ## Этап 8 — Provisioning control plane diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 7b5f5fe..c1aed61 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -5,6 +5,7 @@ IFS=$'\n\t' root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" helper="$root/deploy/ochenstarik-smm-policy-apply" +emergency="$root/deploy/ochenstarik-smm-emergency" help_output="$(bash "$bootstrap" --help)" version_output="$(bash "$bootstrap" --version)" @@ -20,6 +21,9 @@ grep -Fq "node-code NODE_ID" <<<"$help_output" grep -Fq "verify-release ARCHIVE" <<<"$help_output" grep -Fq "node-token NODE_ID" <<<"$help_output" grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output" +emergency_help="$(bash "$emergency" --help)" +grep -Fq 'mesh-disable' <<<"$emergency_help" +grep -Fq 'firewall-restore' <<<"$emergency_help" if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2 @@ -30,6 +34,10 @@ if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22 printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2 exit 1 fi +if bash "$emergency" mesh-disable >/dev/null 2>&1; then + printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2 + exit 1 +fi policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)" printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state" @@ -49,6 +57,7 @@ mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/d install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent" install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control" install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply" +install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency" install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/" install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/"