Add installer reboot and mesh E2E tests
This commit is contained in:
Ochenstarik
parent
3d153aa7b0
commit
5d4e8f063d
6 changed files with 345 additions and 0 deletions
40
.github/workflows/monitor-manager-e2e.yml
vendored
Normal file
40
.github/workflows/monitor-manager-e2e.yml
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
name: Server Monitor Manager E2E
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ['**']
|
||||
paths:
|
||||
- 'ochenstarik-server-monitor-manager.sh'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/monitor-manager-e2e.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'ochenstarik-server-monitor-manager.sh'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/monitor-manager-e2e.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
wireguard-nftables:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install kernel networking tools
|
||||
run: sudo apt-get update && sudo apt-get install -y wireguard-tools nftables iproute2 netcat-openbsd
|
||||
|
||||
- name: Test real WireGuard and nftables data path
|
||||
run: sudo bash tests/e2e/test-monitor-manager-wireguard-nftables.sh
|
||||
|
||||
reinstall-reboot:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test repeated install and systemd reboot
|
||||
run: bash tests/e2e/test-monitor-manager-systemd-reinstall.sh
|
||||
43
.github/workflows/monitor-manager-release.yml
vendored
Normal file
43
.github/workflows/monitor-manager-release.yml
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
name: Server Monitor Manager installer release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- 'smm-installer-v*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
package:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Validate installer
|
||||
run: bash -n ochenstarik-server-monitor-manager.sh
|
||||
|
||||
- name: Create installer checksum
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
install -d dist
|
||||
install -m 0755 ochenstarik-server-monitor-manager.sh dist/ochenstarik-server-monitor-manager.sh
|
||||
cd dist
|
||||
sha256sum ochenstarik-server-monitor-manager.sh > SHA256SUMS
|
||||
sha256sum --check SHA256SUMS
|
||||
|
||||
- name: Upload installer artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ochenstarik-server-monitor-manager-installer
|
||||
path: dist/*
|
||||
|
||||
- name: Attach installer to GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
prerelease: ${{ contains(github.ref_name, '-') }}
|
||||
files: dist/*
|
||||
17
README.md
17
README.md
|
|
@ -53,6 +53,7 @@ The wizard intentionally requires all module scripts to be present locally. Use
|
|||
| `ochenstarik-server-ai-agents-8.sh` | Installs selected AI agents: Hermes, OpenClaw, OpenHands, OpenCode, Aider, AutoGPT, or Pi Coding Agent | Optional |
|
||||
| `ochenstarik-server-monitor-manager.sh` | Installs the Server Monitor Manager SSH endpoint, public WireGuard Hub, or outbound-only Node | Optional; Hub needs a public UDP endpoint |
|
||||
| `ochenstarik-server-uninstall.sh` | Removes project-managed settings so installation can start again | Use carefully |
|
||||
| `ochenstarik-server-monitor-manager.sh` | Installs monitoring-only, Hub, or Node roles for Server Monitor Manager | Yes |
|
||||
|
||||
Every module can be run independently from the full archive or cloned repository:
|
||||
|
||||
|
|
@ -64,6 +65,22 @@ sudo ./SCRIPT_NAME.sh
|
|||
|
||||
Replace `SCRIPT_NAME.sh` with the required filename from the table.
|
||||
|
||||
The Server Monitor Manager installer supports repeatable role-aware commands:
|
||||
|
||||
```bash
|
||||
sudo ./ochenstarik-server-monitor-manager.sh install-monitor
|
||||
sudo ./ochenstarik-server-monitor-manager.sh install-hub
|
||||
sudo ./ochenstarik-server-monitor-manager.sh install-node
|
||||
sudo ./ochenstarik-server-monitor-manager.sh status
|
||||
sudo ./ochenstarik-server-monitor-manager.sh update
|
||||
```
|
||||
|
||||
Release artifacts include `SHA256SUMS`. Verify the installer before running it:
|
||||
|
||||
```bash
|
||||
sha256sum --check SHA256SUMS
|
||||
```
|
||||
|
||||
## Server Monitor Manager Hub and Nodes
|
||||
|
||||
The Hub needs a public IPv4 address or domain and an open UDP port (default `51820`). Secondary Nodes establish outbound WireGuard connections and do not need a public IP.
|
||||
|
|
|
|||
16
tests/e2e/Dockerfile.systemd
Normal file
16
tests/e2e/Dockerfile.systemd
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
FROM debian:bookworm-slim
|
||||
|
||||
ENV container=docker
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends systemd systemd-sysv dbus ca-certificates openssh-client \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& systemctl mask dev-hugepages.mount sys-fs-fuse-connections.mount systemd-remount-fs.service
|
||||
|
||||
COPY . /workspace
|
||||
RUN chmod 700 /workspace/*.sh
|
||||
|
||||
STOPSIGNAL SIGRTMIN+3
|
||||
CMD ["/sbin/init"]
|
||||
80
tests/e2e/test-monitor-manager-systemd-reinstall.sh
Normal file
80
tests/e2e/test-monitor-manager-systemd-reinstall.sh
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
|
||||
|
||||
command -v docker >/dev/null || { echo 'docker is required' >&2; exit 1; }
|
||||
command -v ssh-keygen >/dev/null || { echo 'ssh-keygen is required' >&2; exit 1; }
|
||||
|
||||
run_id="${GITHUB_RUN_ID:-local}-$$"
|
||||
image="smm-systemd-e2e:${run_id}"
|
||||
container="smm-systemd-e2e-${run_id}"
|
||||
temporary="$(mktemp -d)"
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
docker image rm -f "$image" >/dev/null 2>&1 || true
|
||||
rm -rf -- "$temporary"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
ssh-keygen -q -t ed25519 -N '' -C 'smm-e2e' -f "$temporary/monitor-key"
|
||||
public_key="$(<"$temporary/monitor-key.pub")"
|
||||
|
||||
docker build --pull -f tests/e2e/Dockerfile.systemd -t "$image" .
|
||||
docker run -d \
|
||||
--name "$container" \
|
||||
--privileged \
|
||||
--cgroupns=host \
|
||||
--tmpfs /run \
|
||||
--tmpfs /run/lock \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||
"$image" >/dev/null
|
||||
|
||||
for _ in $(seq 1 30); do
|
||||
docker exec "$container" systemctl show --property=Version >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$container" systemctl show --property=Version >/dev/null
|
||||
|
||||
install_hub() {
|
||||
docker exec \
|
||||
-e "SERVER_MONITOR_PUBLIC_KEY=$public_key" \
|
||||
-e SMM_HUB_ENDPOINT=203.0.113.10 \
|
||||
-e SMM_WG_PORT=51820 \
|
||||
"$container" \
|
||||
/workspace/ochenstarik-server-monitor-manager.sh install-hub
|
||||
}
|
||||
|
||||
install_hub
|
||||
first_private_key_hash="$(docker exec "$container" sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')"
|
||||
|
||||
# A repeated install must repair units/configuration without rotating the Hub identity.
|
||||
install_hub
|
||||
second_private_key_hash="$(docker exec "$container" sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')"
|
||||
[[ "$first_private_key_hash" == "$second_private_key_hash" ]]
|
||||
[[ "$(docker exec "$container" grep -c 'ochenstarik-server-monitor' /var/lib/ochenstarik-server-monitor-manager/.ssh/authorized_keys)" -eq 1 ]]
|
||||
|
||||
# Restarting the systemd container is the CI reboot boundary: the writable rootfs is retained,
|
||||
# PID 1 starts again, and only enabled units can reconstruct runtime state.
|
||||
docker restart --timeout 20 "$container" >/dev/null
|
||||
for _ in $(seq 1 45); do
|
||||
if docker exec "$container" systemctl is-active --quiet wg-quick@smm0.service \
|
||||
&& docker exec "$container" systemctl is-active --quiet ochenstarik-smm-firewall.timer \
|
||||
&& docker exec "$container" systemctl is-active --quiet ssh.service; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
docker exec "$container" systemctl is-enabled --quiet wg-quick@smm0.service
|
||||
docker exec "$container" systemctl is-enabled --quiet ochenstarik-smm-firewall.service
|
||||
docker exec "$container" systemctl is-enabled --quiet ochenstarik-smm-firewall.timer
|
||||
docker exec "$container" systemctl is-active --quiet wg-quick@smm0.service
|
||||
docker exec "$container" systemctl is-active --quiet ochenstarik-smm-firewall.timer
|
||||
docker exec "$container" systemctl start ochenstarik-smm-firewall.service
|
||||
docker exec "$container" ip address show dev smm0 | grep -Fq '10.77.0.1/24'
|
||||
docker exec "$container" nft list table inet ochenstarik_smm | grep -Fq 'iifname "smm0" oifname "smm0" drop'
|
||||
docker exec "$container" test "$(sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')" = "$first_private_key_hash"
|
||||
|
||||
printf 'Server Monitor Manager repeated-install and reboot checks passed.\n'
|
||||
149
tests/e2e/test-monitor-manager-wireguard-nftables.sh
Normal file
149
tests/e2e/test-monitor-manager-wireguard-nftables.sh
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
|
||||
|
||||
for command in ip wg nft nc timeout awk sed; do
|
||||
command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 1; }
|
||||
done
|
||||
[[ "$EUID" -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
|
||||
|
||||
temporary="$(mktemp -d)"
|
||||
suffix="${GITHUB_RUN_ID:-$$}"
|
||||
hub="smm-hub-${suffix}"
|
||||
source="smm-source-${suffix}"
|
||||
target="smm-target-${suffix}"
|
||||
listener_pid=''
|
||||
|
||||
cleanup() {
|
||||
[[ -z "$listener_pid" ]] || kill "$listener_pid" >/dev/null 2>&1 || true
|
||||
ip netns delete "$source" >/dev/null 2>&1 || true
|
||||
ip netns delete "$target" >/dev/null 2>&1 || true
|
||||
ip netns delete "$hub" >/dev/null 2>&1 || true
|
||||
rm -rf -- "$temporary"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
for namespace in "$hub" "$source" "$target"; do
|
||||
ip netns add "$namespace"
|
||||
ip -n "$namespace" link set lo up
|
||||
done
|
||||
|
||||
ip link add smm-hub-source type veth peer name smm-source-wan
|
||||
ip link set smm-hub-source netns "$hub"
|
||||
ip link set smm-source-wan netns "$source"
|
||||
ip -n "$hub" address add 192.0.2.1/30 dev smm-hub-source
|
||||
ip -n "$source" address add 192.0.2.2/30 dev smm-source-wan
|
||||
ip -n "$hub" link set smm-hub-source up
|
||||
ip -n "$source" link set smm-source-wan up
|
||||
|
||||
ip link add smm-hub-target type veth peer name smm-target-wan
|
||||
ip link set smm-hub-target netns "$hub"
|
||||
ip link set smm-target-wan netns "$target"
|
||||
ip -n "$hub" address add 192.0.2.5/30 dev smm-hub-target
|
||||
ip -n "$target" address add 192.0.2.6/30 dev smm-target-wan
|
||||
ip -n "$hub" link set smm-hub-target up
|
||||
ip -n "$target" link set smm-target-wan up
|
||||
|
||||
umask 077
|
||||
wg genkey > "$temporary/hub.key"
|
||||
wg genkey > "$temporary/source.key"
|
||||
wg genkey > "$temporary/target.key"
|
||||
wg pubkey < "$temporary/hub.key" > "$temporary/hub.pub"
|
||||
wg pubkey < "$temporary/source.key" > "$temporary/source.pub"
|
||||
wg pubkey < "$temporary/target.key" > "$temporary/target.pub"
|
||||
|
||||
for namespace in "$hub" "$source" "$target"; do
|
||||
ip -n "$namespace" link add smm0 type wireguard
|
||||
done
|
||||
ip netns exec "$hub" wg set smm0 \
|
||||
private-key "$temporary/hub.key" listen-port 51820 \
|
||||
peer "$(<"$temporary/source.pub")" allowed-ips 10.77.0.2/32 \
|
||||
peer "$(<"$temporary/target.pub")" allowed-ips 10.77.0.3/32
|
||||
ip netns exec "$source" wg set smm0 \
|
||||
private-key "$temporary/source.key" \
|
||||
peer "$(<"$temporary/hub.pub")" endpoint 192.0.2.1:51820 allowed-ips 10.77.0.0/24 persistent-keepalive 1
|
||||
ip netns exec "$target" wg set smm0 \
|
||||
private-key "$temporary/target.key" \
|
||||
peer "$(<"$temporary/hub.pub")" endpoint 192.0.2.5:51820 allowed-ips 10.77.0.0/24 persistent-keepalive 1
|
||||
|
||||
ip -n "$hub" address add 10.77.0.1/24 dev smm0
|
||||
ip -n "$source" address add 10.77.0.2/32 dev smm0
|
||||
ip -n "$target" address add 10.77.0.3/32 dev smm0
|
||||
ip -n "$source" route add 10.77.0.0/24 dev smm0
|
||||
ip -n "$target" route add 10.77.0.0/24 dev smm0
|
||||
for namespace in "$hub" "$source" "$target"; do
|
||||
ip -n "$namespace" link set smm0 up
|
||||
done
|
||||
ip netns exec "$hub" sysctl -q -w net.ipv4.ip_forward=1
|
||||
|
||||
helper="$temporary/ochenstarik-smm-hub"
|
||||
awk '
|
||||
capture && /^EOF$/ { exit }
|
||||
/cat > .*HUB_HELPER.*<</ { capture=1; next }
|
||||
capture { print }
|
||||
' ochenstarik-server-monitor-manager.sh > "$helper"
|
||||
chmod 700 "$helper"
|
||||
state="$temporary/state"
|
||||
mkdir -p "$state/nodes" "$state/tokens"
|
||||
sed -i "s|^readonly STATE_DIR=.*|readonly STATE_DIR=\"$state\"|" "$helper"
|
||||
sed -i "s|^readonly WG_CONFIG=.*|readonly WG_CONFIG=\"$temporary/smm0.conf\"|" "$helper"
|
||||
cat > "$state/hub.conf" <<'EOF'
|
||||
HUB_ENDPOINT=203.0.113.10
|
||||
WG_PORT=51820
|
||||
HUB_ADDRESS=10.77.0.1
|
||||
MESH_CIDR=10.77.0.0/24
|
||||
EOF
|
||||
cp "$temporary/hub.key" "$state/hub.key"
|
||||
cat > "$state/nodes/source.node" <<EOF
|
||||
NAME=source
|
||||
ADDRESS=10.77.0.2
|
||||
PUBLIC_KEY=$(<"$temporary/source.pub")
|
||||
EOF
|
||||
cat > "$state/nodes/target.node" <<EOF
|
||||
NAME=target
|
||||
ADDRESS=10.77.0.3
|
||||
PUBLIC_KEY=$(<"$temporary/target.pub")
|
||||
EOF
|
||||
touch "$state/links" "$state/audit.jsonl"
|
||||
printf '0\n' > "$state/policy.version"
|
||||
|
||||
start_listener() {
|
||||
local namespace="$1"
|
||||
ip netns exec "$namespace" sh -c 'printf allowed | timeout 10 nc -l 2222' &
|
||||
listener_pid=$!
|
||||
sleep 0.2
|
||||
}
|
||||
|
||||
ip netns exec "$hub" "$helper" link-connect source target tcp 2222 120
|
||||
start_listener "$target"
|
||||
response="$(timeout 8 ip netns exec "$source" nc -w 5 10.77.0.3 2222)"
|
||||
wait "$listener_pid"
|
||||
listener_pid=''
|
||||
[[ "$response" == allowed ]]
|
||||
|
||||
# The policy is directional: the reverse connection must remain blocked.
|
||||
start_listener "$source"
|
||||
if timeout 3 ip netns exec "$target" nc -w 2 10.77.0.2 2222 >/dev/null 2>&1; then
|
||||
echo 'Reverse traffic bypassed the directional Link.' >&2
|
||||
exit 1
|
||||
fi
|
||||
kill "$listener_pid" >/dev/null 2>&1 || true
|
||||
wait "$listener_pid" 2>/dev/null || true
|
||||
listener_pid=''
|
||||
|
||||
ip netns exec "$hub" "$helper" link-disconnect source target tcp 2222
|
||||
start_listener "$target"
|
||||
if timeout 3 ip netns exec "$source" nc -w 2 10.77.0.3 2222 >/dev/null 2>&1; then
|
||||
echo 'Traffic remained available after the Link kill switch.' >&2
|
||||
exit 1
|
||||
fi
|
||||
kill "$listener_pid" >/dev/null 2>&1 || true
|
||||
wait "$listener_pid" 2>/dev/null || true
|
||||
listener_pid=''
|
||||
|
||||
ip netns exec "$hub" wg show smm0 latest-handshakes | awk '$2 > 0 { found=1 } END { exit !found }'
|
||||
grep -Fq '"state":"Active"' "$state/audit.jsonl"
|
||||
grep -Fq '"state":"Disabled"' "$state/audit.jsonl"
|
||||
|
||||
printf 'Real WireGuard and nftables directional Link checks passed.\n'
|
||||
Loading…
Reference in a new issue