ochenstarik/server-monitor-manager
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases 17 Packages Wiki Activity Actions 2

Add local Mesh emergency recovery

Browse source
This commit is contained in:
Ochenstarik 2026-07-19 14:38:05 +07:00
parent f6192a16e1
commit 470034f137
8 changed files with 175 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/linux-control-agent.yml vendored
View file

@ -41,8 +41,10 @@ jobs:
run: |
bash -n deploy/ochenstarik-server-monitor-manager.sh
bash -n deploy/ochenstarik-smm-policy-apply
bash -n deploy/ochenstarik-smm-emergency
shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh
shellcheck --severity=error deploy/ochenstarik-smm-policy-apply
shellcheck --severity=error deploy/ochenstarik-smm-emergency
bash tests/bootstrap/test-bootstrap-contract.sh
- name: Publish agent amd64
@ -64,6 +66,7 @@ jobs:
install -m 0644 deploy/ochenstarik-smm-agent.service smoke/deploy/
install -m 0644 deploy/ochenstarik-smm-firewall.service smoke/deploy/
install -m 0755 deploy/ochenstarik-smm-policy-apply smoke/deploy/
install -m 0755 deploy/ochenstarik-smm-emergency smoke/deploy/
install -m 0755 deploy/ochenstarik-server-monitor-manager.sh smoke/bootstrap/
tar -C smoke -czf smoke-release.tar.gz agent control deploy bootstrap
sha256sum smoke-release.tar.gz > smoke-release.tar.gz.sha256
@ -90,6 +93,10 @@ jobs:
sudo deploy/ochenstarik-server-monitor-manager.sh install-control \
smoke-release.tar.gz 127.0.0.1 7443 >"$smoke_log" 2>&1 \
|| report_failure 'Initial Control installation failed'
sudo test -x /usr/local/sbin/ochenstarik-smm-emergency \
|| report_failure 'Emergency recovery command was not installed'
sudo /usr/local/sbin/ochenstarik-smm-emergency status >>"$smoke_log" 2>&1 \
|| report_failure 'Emergency recovery status failed'
for _ in {1..30}; do
sudo curl --fail --silent --cacert /etc/ochenstarik-server-monitor-manager/control-ca.crt \
https://127.0.0.1:7443/healthz && break

3
.github/workflows/linux-release.yml vendored
View file

@ -20,8 +20,10 @@ jobs:
run: |
bash -n deploy/ochenstarik-server-monitor-manager.sh
bash -n deploy/ochenstarik-smm-policy-apply
bash -n deploy/ochenstarik-smm-emergency
shellcheck --severity=error deploy/ochenstarik-server-monitor-manager.sh
shellcheck --severity=error deploy/ochenstarik-smm-policy-apply
shellcheck --severity=error deploy/ochenstarik-smm-emergency
bash tests/bootstrap/test-bootstrap-contract.sh
- name: Package bootstrap
@ -87,6 +89,7 @@ jobs:
install -m 0644 deploy/ochenstarik-smm-agent.service out/deploy/
install -m 0644 deploy/ochenstarik-smm-firewall.service out/deploy/
install -m 0755 deploy/ochenstarik-smm-policy-apply out/deploy/
install -m 0755 deploy/ochenstarik-smm-emergency out/deploy/
install -m 0755 deploy/ochenstarik-server-monitor-manager.sh out/bootstrap/
archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz"
tar -C out -czf "$archive" agent control deploy bootstrap

8
deploy/ochenstarik-server-monitor-manager.sh
View file

@ -13,6 +13,7 @@ readonly AGENT_USER="ochenstarik-smm-agent"
readonly CONTROL_UNIT="ochenstarik-smm-control.service"
readonly AGENT_UNIT="ochenstarik-smm-agent.service"
readonly POLICY_HELPER="/usr/local/libexec/ochenstarik-smm-policy-apply"
readonly EMERGENCY_COMMAND="/usr/local/sbin/ochenstarik-smm-emergency"
readonly SUDOERS_FILE="/etc/sudoers.d/ochenstarik-smm-control"
readonly MESH_DIR="${STATE_DIR}/mesh"
readonly WG_DIR="${ETC_DIR}/wireguard"
@ -152,6 +153,8 @@ extract_archive() {
[[ -f "$TEMP_DIR/deploy/$CONTROL_UNIT" ]] || fail "Control systemd unit is missing from archive."
[[ -f "$TEMP_DIR/deploy/$AGENT_UNIT" ]] || fail "Agent systemd unit is missing from archive."
[[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall systemd unit is missing from archive."
[[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing from archive."
[[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency command is missing from archive."
}
verify_release_payload() {
@ -162,6 +165,7 @@ verify_release_payload() {
[[ -x "$TEMP_DIR/control/ochenstarik-smm-control" ]] || fail "Control binary is missing."
[[ -x "$TEMP_DIR/agent/ochenstarik-smm-agent" ]] || fail "Agent binary is missing."
[[ -x "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" ]] || fail "Policy helper is missing."
[[ -x "$TEMP_DIR/deploy/ochenstarik-smm-emergency" ]] || fail "Emergency recovery command is missing."
[[ -f "$TEMP_DIR/deploy/$FIREWALL_UNIT" ]] || fail "Mesh firewall unit is missing."
[[ -x "$TEMP_DIR/bootstrap/ochenstarik-server-monitor-manager.sh" ]] || fail "Packaged bootstrap is missing."
log "Release archive and checksum are valid."
@ -483,6 +487,8 @@ EOF
chmod 0644 "$ETC_DIR/control-public-url"
install -d -m 0755 "$(dirname "$POLICY_HELPER")"
install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-policy-apply" "$POLICY_HELPER"
install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")"
install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND"
install -d -m 0755 "$LIB_DIR/bootstrap"
install -m 0644 "$TEMP_DIR/deploy/$FIREWALL_UNIT" "$LIB_DIR/bootstrap/$FIREWALL_UNIT"
printf '%s\n' "$CONTROL_USER ALL=(root) NOPASSWD: $POLICY_HELPER *" >"$SUDOERS_FILE"
@ -530,6 +536,8 @@ install_agent() {
install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR"
install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent"
install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root"
install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")"
install -m 0755 "$TEMP_DIR/deploy/ochenstarik-smm-emergency" "$EMERGENCY_COMMAND"
if [[ "$(realpath "$ca_cert")" != "$(realpath -m "$ETC_DIR/control-ca.crt")" ]]; then
install -m 0600 -o "$AGENT_USER" -g "$AGENT_USER" "$ca_cert" "$ETC_DIR/control-ca.crt"
else

129
deploy/ochenstarik-smm-emergency Executable file
View file

@ -0,0 +1,129 @@
#!/usr/bin/env bash
set -Eeuo pipefail
IFS=$'\n\t'
readonly PROGRAM="ochenstarik-smm-emergency"
readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager"
readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager"
readonly MARKER="$STATE_DIR/mesh/emergency-disabled"
readonly CONTROL_UNIT="ochenstarik-smm-control.service"
readonly AGENT_UNIT="ochenstarik-smm-agent.service"
readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service"
readonly WIREGUARD_UNIT="wg-quick@smm0.service"
readonly NFT_FAMILY="inet"
readonly NFT_TABLE="ochenstarik_smm"
fail() { printf '%s: %s\n' "$PROGRAM" "$*" >&2; exit 78; }
log() { printf '%s: %s\n' "$PROGRAM" "$*"; }
usage() {
cat <<'EOF'
Local Server Monitor Manager emergency recovery
Usage:
ochenstarik-smm-emergency status
ochenstarik-smm-emergency mesh-disable
ochenstarik-smm-emergency mesh-enable
ochenstarik-smm-emergency firewall-restore
Commands use only Server Monitor Manager-owned units, interface and nftables
table. They do not modify the host SSH service or unrelated firewall rules.
EOF
}
require_root() {
[[ ${EUID:-$(id -u)} -eq 0 ]] || fail "this action must run as root (use sudo)"
}
unit_state() {
local unit="$1"
if systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q "^$unit"; then
systemctl is-active "$unit" 2>/dev/null || true
else
printf '%s\n' "not-installed"
fi
}
show_status() {
local unit
for unit in "$CONTROL_UNIT" "$AGENT_UNIT" "$FIREWALL_UNIT" "$WIREGUARD_UNIT"; do
printf '%s: %s\n' "$unit" "$(unit_state "$unit")"
done
if command -v ip >/dev/null 2>&1 && ip link show smm0 >/dev/null 2>&1; then
printf '%s\n' "mesh-interface: present"
else
printf '%s\n' "mesh-interface: absent"
fi
if command -v nft >/dev/null 2>&1 && nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then
printf '%s\n' "mesh-firewall: loaded"
else
printf '%s\n' "mesh-firewall: absent"
fi
[[ -f "$MARKER" ]] && printf '%s\n' "emergency-lock: active" || printf '%s\n' "emergency-lock: inactive"
printf '%s\n' "backups:"
find "$STATE_DIR/bootstrap-backups" -maxdepth 1 -type f \
\( -name '*.tar.gz' -o -name '*.empty' \) -printf ' %f\n' 2>/dev/null | sort -r | head -n 10 || true
}
delete_project_firewall() {
if command -v nft >/dev/null 2>&1; then
nft delete table "$NFT_FAMILY" "$NFT_TABLE" 2>/dev/null || true
fi
}
mesh_disable() {
require_root
systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
systemctl disable --now "$FIREWALL_UNIT" 2>/dev/null || true
delete_project_firewall
install -d -m 0700 "$(dirname "$MARKER")"
printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$MARKER"
chmod 0600 "$MARKER"
log "Mesh disabled locally; Control and SSH were not changed."
}
restore_project_firewall() {
require_root
[[ -f "$ETC_DIR/mesh.nft" ]] || fail "managed firewall configuration is missing"
command -v nft >/dev/null 2>&1 || fail "nft is unavailable"
if ! nft list table "$NFT_FAMILY" "$NFT_TABLE" >/dev/null 2>&1; then
nft --check -f "$ETC_DIR/mesh.nft"
fi
delete_project_firewall
if ! nft -f "$ETC_DIR/mesh.nft"; then
systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
fail "managed firewall restore failed; Mesh was disabled to fail closed"
fi
if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then
systemctl enable "$FIREWALL_UNIT" >/dev/null
fi
log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links."
}
mesh_enable() {
require_root
[[ -f /etc/wireguard/smm0.conf ]] || fail "WireGuard smm0 configuration is missing"
if [[ -f "$ETC_DIR/mesh.nft" ]]; then
restore_project_firewall
fi
systemctl enable "$WIREGUARD_UNIT" >/dev/null
systemctl restart "$WIREGUARD_UNIT"
rm -f -- "$MARKER"
log "Mesh enabled locally."
}
main() {
local action="${1:-help}"
shift || true
[[ $# -eq 0 ]] || fail "unexpected arguments"
case "$action" in
help|-h|--help) usage ;;
status) show_status ;;
mesh-disable) mesh_disable ;;
mesh-enable) mesh_enable ;;
firewall-restore) restore_project_firewall ;;
*) fail "unknown action: $action" ;;
esac
}
main "$@"

8
docs/installer-contract.md
View file

@ -82,11 +82,13 @@ control device-code DEVICE_ID
control node-code NODE_ID
control automation-token AUTOMATION_ID SOURCE_NODE_ID
emergency status
emergency vpn-disable
emergency ssh-restore BACKUP_ID
emergency firewall-restore BACKUP_ID
emergency mesh-disable
emergency mesh-enable
emergency firewall-restore
```
Текущая emergency-команда покрывает уже реализованные Mesh/WireGuard и project-owned nftables resources. Действия `vpn-disable` и `ssh-restore BACKUP_ID` добавляются вместе с соответствующими provisioning-модулями и их root-only backup format; до появления этих модулей команда намеренно их отклоняет.
CLI является non-interactive, кроме локального ввода enrollment code и явных подтверждений опасного удаления. Машиночитаемый режим возвращает versioned JSON и стабильные exit codes.
## 6. Идемпотентность и обновление

13
docs/linux-bootstrap.md
View file

@ -96,3 +96,16 @@ sudo ./ochenstarik-server-monitor-manager.sh uninstall-control --confirm-destroy
```
Update создаёт root-only backup перед заменой binaries и автоматически восстанавливает предыдущую версию, если сервис не запускается. Перед alpha-тестом на реальных серверах обязательно сохраните отдельную консольную/SSH-сессию и не закрывайте основной административный доступ firewall-правилами проекта.
## Локальное аварийное восстановление
Установщик размещает независимую от Control Hub команду `/usr/local/sbin/ochenstarik-smm-emergency`. Она принимает только фиксированные действия и управляет исключительно интерфейсом `smm0`, systemd units и nftables-таблицей Server Monitor Manager:
```bash
sudo ochenstarik-smm-emergency status
sudo ochenstarik-smm-emergency mesh-disable
sudo ochenstarik-smm-emergency firewall-restore
sudo ochenstarik-smm-emergency mesh-enable
```
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub.

2
docs/roadmap.md
View file

@ -100,7 +100,7 @@
- [x] добавить собственную установку WireGuard Hub/Node и выдачу внутренних адресов;
- [x] реализовать nftables policy helper вместо временного deny-by-default helper;
- [ ] добавить VM CI matrix, повторную установку и reboot;
- [ ] добавить локальную emergency recovery command.
- [x] добавить локальную emergency recovery command для текущих Mesh/firewall-компонентов.
## Этап 8 — Provisioning control plane

9
tests/bootstrap/test-bootstrap-contract.sh
View file

@ -5,6 +5,7 @@ IFS=$'\n\t'
root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
helper="$root/deploy/ochenstarik-smm-policy-apply"
emergency="$root/deploy/ochenstarik-smm-emergency"
help_output="$(bash "$bootstrap" --help)"
version_output="$(bash "$bootstrap" --version)"
@ -20,6 +21,9 @@ grep -Fq "node-code NODE_ID" <<<"$help_output"
grep -Fq "verify-release ARCHIVE" <<<"$help_output"
grep -Fq "node-token NODE_ID" <<<"$help_output"
grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$version_output"
emergency_help="$(bash "$emergency" --help)"
grep -Fq 'mesh-disable' <<<"$emergency_help"
grep -Fq 'firewall-restore' <<<"$emergency_help"
if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then
printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2
@ -30,6 +34,10 @@ if env -u SUDO_UID -u SUDO_USER bash "$helper" link-connect source target tcp 22
printf '%s\n' "policy helper unexpectedly applied an unconfigured rule" >&2
exit 1
fi
if bash "$emergency" mesh-disable >/dev/null 2>&1; then
printf '%s\n' "emergency mutation unexpectedly succeeded without root" >&2
exit 1
fi
policy_state="$(mktemp -t smm-policy-state.XXXXXXXX)"
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$policy_state"
@ -49,6 +57,7 @@ mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/d
install -m 0755 /bin/true "$fixture/payload/agent/ochenstarik-smm-agent"
install -m 0755 /bin/true "$fixture/payload/control/ochenstarik-smm-control"
install -m 0755 "$helper" "$fixture/payload/deploy/ochenstarik-smm-policy-apply"
install -m 0755 "$emergency" "$fixture/payload/deploy/ochenstarik-smm-emergency"
install -m 0644 "$root/deploy/ochenstarik-smm-control.service" "$fixture/payload/deploy/"
install -m 0644 "$root/deploy/ochenstarik-smm-agent.service" "$fixture/payload/deploy/"
install -m 0644 "$root/deploy/ochenstarik-smm-firewall.service" "$fixture/payload/deploy/"