Integrate Server Monitor installer and real E2E tests #17
6 changed files with 345 additions and 0 deletions
40
.github/workflows/monitor-manager-e2e.yml
vendored
Normal file
40
.github/workflows/monitor-manager-e2e.yml
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
name: Server Monitor Manager E2E
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ['**']
|
||||||
|
paths:
|
||||||
|
- 'ochenstarik-server-monitor-manager.sh'
|
||||||
|
- 'tests/e2e/**'
|
||||||
|
- '.github/workflows/monitor-manager-e2e.yml'
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- 'ochenstarik-server-monitor-manager.sh'
|
||||||
|
- 'tests/e2e/**'
|
||||||
|
- '.github/workflows/monitor-manager-e2e.yml'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
wireguard-nftables:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install kernel networking tools
|
||||||
|
run: sudo apt-get update && sudo apt-get install -y wireguard-tools nftables iproute2 netcat-openbsd
|
||||||
|
|
||||||
|
- name: Test real WireGuard and nftables data path
|
||||||
|
run: sudo bash tests/e2e/test-monitor-manager-wireguard-nftables.sh
|
||||||
|
|
||||||
|
reinstall-reboot:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Test repeated install and systemd reboot
|
||||||
|
run: bash tests/e2e/test-monitor-manager-systemd-reinstall.sh
|
||||||
43
.github/workflows/monitor-manager-release.yml
vendored
Normal file
43
.github/workflows/monitor-manager-release.yml
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
name: Server Monitor Manager installer release
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'smm-installer-v*'
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
package:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Validate installer
|
||||||
|
run: bash -n ochenstarik-server-monitor-manager.sh
|
||||||
|
|
||||||
|
- name: Create installer checksum
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -Eeuo pipefail
|
||||||
|
install -d dist
|
||||||
|
install -m 0755 ochenstarik-server-monitor-manager.sh dist/ochenstarik-server-monitor-manager.sh
|
||||||
|
cd dist
|
||||||
|
sha256sum ochenstarik-server-monitor-manager.sh > SHA256SUMS
|
||||||
|
sha256sum --check SHA256SUMS
|
||||||
|
|
||||||
|
- name: Upload installer artifact
|
||||||
|
uses: actions/upload-artifact@v6
|
||||||
|
with:
|
||||||
|
name: ochenstarik-server-monitor-manager-installer
|
||||||
|
path: dist/*
|
||||||
|
|
||||||
|
- name: Attach installer to GitHub Release
|
||||||
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
uses: softprops/action-gh-release@v2
|
||||||
|
with:
|
||||||
|
prerelease: ${{ contains(github.ref_name, '-') }}
|
||||||
|
files: dist/*
|
||||||
17
README.md
17
README.md
|
|
@ -53,6 +53,7 @@ The wizard intentionally requires all module scripts to be present locally. Use
|
||||||
| `ochenstarik-server-ai-agents-8.sh` | Installs selected AI agents: Hermes, OpenClaw, OpenHands, OpenCode, Aider, AutoGPT, or Pi Coding Agent | Optional |
|
| `ochenstarik-server-ai-agents-8.sh` | Installs selected AI agents: Hermes, OpenClaw, OpenHands, OpenCode, Aider, AutoGPT, or Pi Coding Agent | Optional |
|
||||||
| `ochenstarik-server-monitor-manager.sh` | Installs the Server Monitor Manager SSH endpoint, public WireGuard Hub, or outbound-only Node | Optional; Hub needs a public UDP endpoint |
|
| `ochenstarik-server-monitor-manager.sh` | Installs the Server Monitor Manager SSH endpoint, public WireGuard Hub, or outbound-only Node | Optional; Hub needs a public UDP endpoint |
|
||||||
| `ochenstarik-server-uninstall.sh` | Removes project-managed settings so installation can start again | Use carefully |
|
| `ochenstarik-server-uninstall.sh` | Removes project-managed settings so installation can start again | Use carefully |
|
||||||
|
| `ochenstarik-server-monitor-manager.sh` | Installs monitoring-only, Hub, or Node roles for Server Monitor Manager | Yes |
|
||||||
|
|
||||||
Every module can be run independently from the full archive or cloned repository:
|
Every module can be run independently from the full archive or cloned repository:
|
||||||
|
|
||||||
|
|
@ -64,6 +65,22 @@ sudo ./SCRIPT_NAME.sh
|
||||||
|
|
||||||
Replace `SCRIPT_NAME.sh` with the required filename from the table.
|
Replace `SCRIPT_NAME.sh` with the required filename from the table.
|
||||||
|
|
||||||
|
The Server Monitor Manager installer supports repeatable role-aware commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./ochenstarik-server-monitor-manager.sh install-monitor
|
||||||
|
sudo ./ochenstarik-server-monitor-manager.sh install-hub
|
||||||
|
sudo ./ochenstarik-server-monitor-manager.sh install-node
|
||||||
|
sudo ./ochenstarik-server-monitor-manager.sh status
|
||||||
|
sudo ./ochenstarik-server-monitor-manager.sh update
|
||||||
|
```
|
||||||
|
|
||||||
|
Release artifacts include `SHA256SUMS`. Verify the installer before running it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sha256sum --check SHA256SUMS
|
||||||
|
```
|
||||||
|
|
||||||
## Server Monitor Manager Hub and Nodes
|
## Server Monitor Manager Hub and Nodes
|
||||||
|
|
||||||
The Hub needs a public IPv4 address or domain and an open UDP port (default `51820`). Secondary Nodes establish outbound WireGuard connections and do not need a public IP.
|
The Hub needs a public IPv4 address or domain and an open UDP port (default `51820`). Secondary Nodes establish outbound WireGuard connections and do not need a public IP.
|
||||||
|
|
|
||||||
16
tests/e2e/Dockerfile.systemd
Normal file
16
tests/e2e/Dockerfile.systemd
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
ENV container=docker
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends systemd systemd-sysv dbus ca-certificates openssh-client \
|
||||||
|
&& apt-get clean \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& systemctl mask dev-hugepages.mount sys-fs-fuse-connections.mount systemd-remount-fs.service
|
||||||
|
|
||||||
|
COPY . /workspace
|
||||||
|
RUN chmod 700 /workspace/*.sh
|
||||||
|
|
||||||
|
STOPSIGNAL SIGRTMIN+3
|
||||||
|
CMD ["/sbin/init"]
|
||||||
80
tests/e2e/test-monitor-manager-systemd-reinstall.sh
Normal file
80
tests/e2e/test-monitor-manager-systemd-reinstall.sh
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
|
||||||
|
|
||||||
|
command -v docker >/dev/null || { echo 'docker is required' >&2; exit 1; }
|
||||||
|
command -v ssh-keygen >/dev/null || { echo 'ssh-keygen is required' >&2; exit 1; }
|
||||||
|
|
||||||
|
run_id="${GITHUB_RUN_ID:-local}-$$"
|
||||||
|
image="smm-systemd-e2e:${run_id}"
|
||||||
|
container="smm-systemd-e2e-${run_id}"
|
||||||
|
temporary="$(mktemp -d)"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||||
|
docker image rm -f "$image" >/dev/null 2>&1 || true
|
||||||
|
rm -rf -- "$temporary"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
ssh-keygen -q -t ed25519 -N '' -C 'smm-e2e' -f "$temporary/monitor-key"
|
||||||
|
public_key="$(<"$temporary/monitor-key.pub")"
|
||||||
|
|
||||||
|
docker build --pull -f tests/e2e/Dockerfile.systemd -t "$image" .
|
||||||
|
docker run -d \
|
||||||
|
--name "$container" \
|
||||||
|
--privileged \
|
||||||
|
--cgroupns=host \
|
||||||
|
--tmpfs /run \
|
||||||
|
--tmpfs /run/lock \
|
||||||
|
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||||
|
"$image" >/dev/null
|
||||||
|
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
docker exec "$container" systemctl show --property=Version >/dev/null 2>&1 && break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
docker exec "$container" systemctl show --property=Version >/dev/null
|
||||||
|
|
||||||
|
install_hub() {
|
||||||
|
docker exec \
|
||||||
|
-e "SERVER_MONITOR_PUBLIC_KEY=$public_key" \
|
||||||
|
-e SMM_HUB_ENDPOINT=203.0.113.10 \
|
||||||
|
-e SMM_WG_PORT=51820 \
|
||||||
|
"$container" \
|
||||||
|
/workspace/ochenstarik-server-monitor-manager.sh install-hub
|
||||||
|
}
|
||||||
|
|
||||||
|
install_hub
|
||||||
|
first_private_key_hash="$(docker exec "$container" sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')"
|
||||||
|
|
||||||
|
# A repeated install must repair units/configuration without rotating the Hub identity.
|
||||||
|
install_hub
|
||||||
|
second_private_key_hash="$(docker exec "$container" sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')"
|
||||||
|
[[ "$first_private_key_hash" == "$second_private_key_hash" ]]
|
||||||
|
[[ "$(docker exec "$container" grep -c 'ochenstarik-server-monitor' /var/lib/ochenstarik-server-monitor-manager/.ssh/authorized_keys)" -eq 1 ]]
|
||||||
|
|
||||||
|
# Restarting the systemd container is the CI reboot boundary: the writable rootfs is retained,
|
||||||
|
# PID 1 starts again, and only enabled units can reconstruct runtime state.
|
||||||
|
docker restart --timeout 20 "$container" >/dev/null
|
||||||
|
for _ in $(seq 1 45); do
|
||||||
|
if docker exec "$container" systemctl is-active --quiet wg-quick@smm0.service \
|
||||||
|
&& docker exec "$container" systemctl is-active --quiet ochenstarik-smm-firewall.timer \
|
||||||
|
&& docker exec "$container" systemctl is-active --quiet ssh.service; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
docker exec "$container" systemctl is-enabled --quiet wg-quick@smm0.service
|
||||||
|
docker exec "$container" systemctl is-enabled --quiet ochenstarik-smm-firewall.service
|
||||||
|
docker exec "$container" systemctl is-enabled --quiet ochenstarik-smm-firewall.timer
|
||||||
|
docker exec "$container" systemctl is-active --quiet wg-quick@smm0.service
|
||||||
|
docker exec "$container" systemctl is-active --quiet ochenstarik-smm-firewall.timer
|
||||||
|
docker exec "$container" systemctl start ochenstarik-smm-firewall.service
|
||||||
|
docker exec "$container" ip address show dev smm0 | grep -Fq '10.77.0.1/24'
|
||||||
|
docker exec "$container" nft list table inet ochenstarik_smm | grep -Fq 'iifname "smm0" oifname "smm0" drop'
|
||||||
|
docker exec "$container" test "$(sha256sum /etc/ochenstarik-server-monitor-manager/hub.key | awk '{ print $1 }')" = "$first_private_key_hash"
|
||||||
|
|
||||||
|
printf 'Server Monitor Manager repeated-install and reboot checks passed.\n'
|
||||||
149
tests/e2e/test-monitor-manager-wireguard-nftables.sh
Normal file
149
tests/e2e/test-monitor-manager-wireguard-nftables.sh
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
|
||||||
|
|
||||||
|
for command in ip wg nft nc timeout awk sed; do
|
||||||
|
command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
[[ "$EUID" -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
|
||||||
|
|
||||||
|
temporary="$(mktemp -d)"
|
||||||
|
suffix="${GITHUB_RUN_ID:-$$}"
|
||||||
|
hub="smm-hub-${suffix}"
|
||||||
|
source="smm-source-${suffix}"
|
||||||
|
target="smm-target-${suffix}"
|
||||||
|
listener_pid=''
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
[[ -z "$listener_pid" ]] || kill "$listener_pid" >/dev/null 2>&1 || true
|
||||||
|
ip netns delete "$source" >/dev/null 2>&1 || true
|
||||||
|
ip netns delete "$target" >/dev/null 2>&1 || true
|
||||||
|
ip netns delete "$hub" >/dev/null 2>&1 || true
|
||||||
|
rm -rf -- "$temporary"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
for namespace in "$hub" "$source" "$target"; do
|
||||||
|
ip netns add "$namespace"
|
||||||
|
ip -n "$namespace" link set lo up
|
||||||
|
done
|
||||||
|
|
||||||
|
ip link add smm-hub-source type veth peer name smm-source-wan
|
||||||
|
ip link set smm-hub-source netns "$hub"
|
||||||
|
ip link set smm-source-wan netns "$source"
|
||||||
|
ip -n "$hub" address add 192.0.2.1/30 dev smm-hub-source
|
||||||
|
ip -n "$source" address add 192.0.2.2/30 dev smm-source-wan
|
||||||
|
ip -n "$hub" link set smm-hub-source up
|
||||||
|
ip -n "$source" link set smm-source-wan up
|
||||||
|
|
||||||
|
ip link add smm-hub-target type veth peer name smm-target-wan
|
||||||
|
ip link set smm-hub-target netns "$hub"
|
||||||
|
ip link set smm-target-wan netns "$target"
|
||||||
|
ip -n "$hub" address add 192.0.2.5/30 dev smm-hub-target
|
||||||
|
ip -n "$target" address add 192.0.2.6/30 dev smm-target-wan
|
||||||
|
ip -n "$hub" link set smm-hub-target up
|
||||||
|
ip -n "$target" link set smm-target-wan up
|
||||||
|
|
||||||
|
umask 077
|
||||||
|
wg genkey > "$temporary/hub.key"
|
||||||
|
wg genkey > "$temporary/source.key"
|
||||||
|
wg genkey > "$temporary/target.key"
|
||||||
|
wg pubkey < "$temporary/hub.key" > "$temporary/hub.pub"
|
||||||
|
wg pubkey < "$temporary/source.key" > "$temporary/source.pub"
|
||||||
|
wg pubkey < "$temporary/target.key" > "$temporary/target.pub"
|
||||||
|
|
||||||
|
for namespace in "$hub" "$source" "$target"; do
|
||||||
|
ip -n "$namespace" link add smm0 type wireguard
|
||||||
|
done
|
||||||
|
ip netns exec "$hub" wg set smm0 \
|
||||||
|
private-key "$temporary/hub.key" listen-port 51820 \
|
||||||
|
peer "$(<"$temporary/source.pub")" allowed-ips 10.77.0.2/32 \
|
||||||
|
peer "$(<"$temporary/target.pub")" allowed-ips 10.77.0.3/32
|
||||||
|
ip netns exec "$source" wg set smm0 \
|
||||||
|
private-key "$temporary/source.key" \
|
||||||
|
peer "$(<"$temporary/hub.pub")" endpoint 192.0.2.1:51820 allowed-ips 10.77.0.0/24 persistent-keepalive 1
|
||||||
|
ip netns exec "$target" wg set smm0 \
|
||||||
|
private-key "$temporary/target.key" \
|
||||||
|
peer "$(<"$temporary/hub.pub")" endpoint 192.0.2.5:51820 allowed-ips 10.77.0.0/24 persistent-keepalive 1
|
||||||
|
|
||||||
|
ip -n "$hub" address add 10.77.0.1/24 dev smm0
|
||||||
|
ip -n "$source" address add 10.77.0.2/32 dev smm0
|
||||||
|
ip -n "$target" address add 10.77.0.3/32 dev smm0
|
||||||
|
ip -n "$source" route add 10.77.0.0/24 dev smm0
|
||||||
|
ip -n "$target" route add 10.77.0.0/24 dev smm0
|
||||||
|
for namespace in "$hub" "$source" "$target"; do
|
||||||
|
ip -n "$namespace" link set smm0 up
|
||||||
|
done
|
||||||
|
ip netns exec "$hub" sysctl -q -w net.ipv4.ip_forward=1
|
||||||
|
|
||||||
|
helper="$temporary/ochenstarik-smm-hub"
|
||||||
|
awk '
|
||||||
|
capture && /^EOF$/ { exit }
|
||||||
|
/cat > .*HUB_HELPER.*<</ { capture=1; next }
|
||||||
|
capture { print }
|
||||||
|
' ochenstarik-server-monitor-manager.sh > "$helper"
|
||||||
|
chmod 700 "$helper"
|
||||||
|
state="$temporary/state"
|
||||||
|
mkdir -p "$state/nodes" "$state/tokens"
|
||||||
|
sed -i "s|^readonly STATE_DIR=.*|readonly STATE_DIR=\"$state\"|" "$helper"
|
||||||
|
sed -i "s|^readonly WG_CONFIG=.*|readonly WG_CONFIG=\"$temporary/smm0.conf\"|" "$helper"
|
||||||
|
cat > "$state/hub.conf" <<'EOF'
|
||||||
|
HUB_ENDPOINT=203.0.113.10
|
||||||
|
WG_PORT=51820
|
||||||
|
HUB_ADDRESS=10.77.0.1
|
||||||
|
MESH_CIDR=10.77.0.0/24
|
||||||
|
EOF
|
||||||
|
cp "$temporary/hub.key" "$state/hub.key"
|
||||||
|
cat > "$state/nodes/source.node" <<EOF
|
||||||
|
NAME=source
|
||||||
|
ADDRESS=10.77.0.2
|
||||||
|
PUBLIC_KEY=$(<"$temporary/source.pub")
|
||||||
|
EOF
|
||||||
|
cat > "$state/nodes/target.node" <<EOF
|
||||||
|
NAME=target
|
||||||
|
ADDRESS=10.77.0.3
|
||||||
|
PUBLIC_KEY=$(<"$temporary/target.pub")
|
||||||
|
EOF
|
||||||
|
touch "$state/links" "$state/audit.jsonl"
|
||||||
|
printf '0\n' > "$state/policy.version"
|
||||||
|
|
||||||
|
start_listener() {
|
||||||
|
local namespace="$1"
|
||||||
|
ip netns exec "$namespace" sh -c 'printf allowed | timeout 10 nc -l 2222' &
|
||||||
|
listener_pid=$!
|
||||||
|
sleep 0.2
|
||||||
|
}
|
||||||
|
|
||||||
|
ip netns exec "$hub" "$helper" link-connect source target tcp 2222 120
|
||||||
|
start_listener "$target"
|
||||||
|
response="$(timeout 8 ip netns exec "$source" nc -w 5 10.77.0.3 2222)"
|
||||||
|
wait "$listener_pid"
|
||||||
|
listener_pid=''
|
||||||
|
[[ "$response" == allowed ]]
|
||||||
|
|
||||||
|
# The policy is directional: the reverse connection must remain blocked.
|
||||||
|
start_listener "$source"
|
||||||
|
if timeout 3 ip netns exec "$target" nc -w 2 10.77.0.2 2222 >/dev/null 2>&1; then
|
||||||
|
echo 'Reverse traffic bypassed the directional Link.' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
kill "$listener_pid" >/dev/null 2>&1 || true
|
||||||
|
wait "$listener_pid" 2>/dev/null || true
|
||||||
|
listener_pid=''
|
||||||
|
|
||||||
|
ip netns exec "$hub" "$helper" link-disconnect source target tcp 2222
|
||||||
|
start_listener "$target"
|
||||||
|
if timeout 3 ip netns exec "$source" nc -w 2 10.77.0.3 2222 >/dev/null 2>&1; then
|
||||||
|
echo 'Traffic remained available after the Link kill switch.' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
kill "$listener_pid" >/dev/null 2>&1 || true
|
||||||
|
wait "$listener_pid" 2>/dev/null || true
|
||||||
|
listener_pid=''
|
||||||
|
|
||||||
|
ip netns exec "$hub" wg show smm0 latest-handshakes | awk '$2 > 0 { found=1 } END { exit !found }'
|
||||||
|
grep -Fq '"state":"Active"' "$state/audit.jsonl"
|
||||||
|
grep -Fq '"state":"Disabled"' "$state/audit.jsonl"
|
||||||
|
|
||||||
|
printf 'Real WireGuard and nftables directional Link checks passed.\n'
|
||||||
Loading…
Reference in a new issue