From 2fe570afd8b4d729875ff7ca5b794fcbba5bff2f Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Fri, 17 Jul 2026 00:38:54 +0700 Subject: [PATCH] Test kill switch helper failures --- README.md | 2 +- docs/installer-contract.md | 2 + docs/roadmap.md | 3 +- .../ControlOptions.cs | 2 + .../LinkPolicyApplier.cs | 2 +- src/ServerMonitorManager.Control/Program.cs | 4 +- .../appsettings.json | 3 +- .../LinkPolicyApplierIntegrationTests.cs | 169 ++++++++++++++++++ 8 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 tests/ServerMonitorManager.Control.Tests/LinkPolicyApplierIntegrationTests.cs diff --git a/README.md b/README.md index 21bd5b7..f84d555 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements Windows SSH monitoring, the Hub/Node WireGuard installer, directional Links, one-time enrollment, mTLS Agent and Operator identities, certificate revocation/re-enrollment, SQLite control state, audit, authenticated event streaming, Windows Control API integration, and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Still planned: kill-switch integration tests, a 50–100 Node load test, signed Windows installer, and desktop/mobile clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Linux CI exercises the real Control-to-helper process boundary, including a helper failure and Control process reconstruction over the same SQLite database. Still planned: end-to-end nftables and host-reboot tests with the installer, a 50–100 Node load test, signed Windows installer, and desktop/mobile clients for additional platforms. ## License and project policy diff --git a/docs/installer-contract.md b/docs/installer-contract.md index 33165cd..a8d4b58 100644 --- a/docs/installer-contract.md +++ b/docs/installer-contract.md @@ -51,6 +51,8 @@ Control service не получает общий доступ к root helper. Отдельный root-owned wrapper принимает только проверенные `link-connect` и `link-disconnect`; команды регистрации, удаления Node и произвольные аргументы ему недоступны. +Репозиторий Control проверяет границу запуска helper отдельным Linux integration test: реальный дочерний процесс, обязательный non-interactive privilege wrapper, сохранение `Disabled/Partial` в SQLite и восстановление после пересоздания Control process. Проверка настоящих nftables ruleset и reboot хоста выполняется вместе с исходным установщиком, без его копирования в этот репозиторий. + ## Команды жизненного цикла Целевой интерфейс: diff --git a/docs/roadmap.md b/docs/roadmap.md index bf6e801..a524b5a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -54,7 +54,8 @@ - [x] версия политики и подтверждение применения на Hub; - [x] обязательное отключение после reconnect; - [x] локальный append-only JSONL-аудит операций Link; -- [ ] интеграционные тесты kill switch и частичных отказов. +- [x] интеграционные тесты Control kill switch, перезапуска процесса и частичного отказа helper; +- [ ] end-to-end тесты nftables и реального reboot вместе с Linux-установщиком. ## Этап 5 — мониторинг и терминал diff --git a/src/ServerMonitorManager.Control/ControlOptions.cs b/src/ServerMonitorManager.Control/ControlOptions.cs index 6618199..d67a63c 100644 --- a/src/ServerMonitorManager.Control/ControlOptions.cs +++ b/src/ServerMonitorManager.Control/ControlOptions.cs @@ -15,4 +15,6 @@ public sealed class ControlOptions public int MaxBufferedMetricAgeHours { get; init; } = 24; public string HubHelperPath { get; init; } = "/usr/local/libexec/ochenstarik-smm-policy-apply"; + + public string PrivilegeEscalationPath { get; init; } = "/usr/bin/sudo"; } diff --git a/src/ServerMonitorManager.Control/LinkPolicyApplier.cs b/src/ServerMonitorManager.Control/LinkPolicyApplier.cs index 44e0388..5f1fdf8 100644 --- a/src/ServerMonitorManager.Control/LinkPolicyApplier.cs +++ b/src/ServerMonitorManager.Control/LinkPolicyApplier.cs @@ -39,7 +39,7 @@ public sealed class LinkPolicyApplier(IOptions options) : ILinkP { var startInfo = new ProcessStartInfo { - FileName = "/usr/bin/sudo", + FileName = options.Value.PrivilegeEscalationPath, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index c52e469..a128fb9 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -34,9 +34,11 @@ builder.Services.AddOptions() .Validate(options => !string.IsNullOrWhiteSpace(options.DatabasePath) && !string.IsNullOrWhiteSpace(options.CertificateAuthorityPath) + && !string.IsNullOrWhiteSpace(options.HubHelperPath) + && !string.IsNullOrWhiteSpace(options.PrivilegeEscalationPath) && options.HeartbeatSeconds is >= 10 and <= 300 && options.MaxBufferedMetricAgeHours is >= 1 and <= 168, - "Control paths are required, HeartbeatSeconds must be 10-300, and buffered metrics 1-168 hours.") + "Control, helper, and privilege escalation paths are required; HeartbeatSeconds must be 10-300, and buffered metrics 1-168 hours.") .ValidateOnStart(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/ServerMonitorManager.Control/appsettings.json b/src/ServerMonitorManager.Control/appsettings.json index 99843cf..df891f1 100644 --- a/src/ServerMonitorManager.Control/appsettings.json +++ b/src/ServerMonitorManager.Control/appsettings.json @@ -5,7 +5,8 @@ "CertificateAuthorityPassword": null, "HeartbeatSeconds": 30, "MaxBufferedMetricAgeHours": 24, - "HubHelperPath": "/usr/local/libexec/ochenstarik-smm-policy-apply" + "HubHelperPath": "/usr/local/libexec/ochenstarik-smm-policy-apply", + "PrivilegeEscalationPath": "/usr/bin/sudo" }, "Logging": { "LogLevel": { diff --git a/tests/ServerMonitorManager.Control.Tests/LinkPolicyApplierIntegrationTests.cs b/tests/ServerMonitorManager.Control.Tests/LinkPolicyApplierIntegrationTests.cs new file mode 100644 index 0000000..8f6f2d4 --- /dev/null +++ b/tests/ServerMonitorManager.Control.Tests/LinkPolicyApplierIntegrationTests.cs @@ -0,0 +1,169 @@ +using System.Runtime.Versioning; +using System.Text; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Options; +using ServerMonitorManager.Control; +using ServerMonitorManager.Core; +using Xunit; + +namespace ServerMonitorManager.Control.Tests; + +public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable +{ + private readonly string _directory = Path.Combine( + Path.GetTempPath(), $"smm-helper-tests-{Guid.NewGuid():N}"); + + [Fact] + public async Task LinuxHelperFailureKeepsKillSwitchPendingAcrossControlRestart() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + var cancellationToken = TestContext.Current.CancellationToken; + Directory.CreateDirectory(_directory); + var sudoPath = Path.Combine(_directory, "sudo"); + var helperPath = Path.Combine(_directory, "policy-helper"); + var failureMarkerPath = Path.Combine(_directory, "fail-disconnect"); + var invocationLogPath = Path.Combine(_directory, "helper.log"); + await WriteExecutableAsync( + sudoPath, + """ + #!/bin/sh + set -eu + if [ "${1:-}" != "-n" ]; then + echo "sudo must be non-interactive" >&2 + exit 90 + fi + shift + exec "$@" + """, + cancellationToken); + await WriteExecutableAsync( + helperPath, + $$""" + #!/bin/sh + set -eu + printf '%s\n' "$*" >> '{{ShellQuote(invocationLogPath)}}' + if [ "${1:-}" = "link-disconnect" ] && [ -f '{{ShellQuote(failureMarkerPath)}}' ]; then + echo "nftables validation failed" >&2 + exit 23 + fi + """, + cancellationToken); + + var store = CreateStore(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "ai-agent", "AABB", cancellationToken); + await EnrollAgentAsync(store, "home", "CCDD", cancellationToken); + var service = CreateLinkService(store, sudoPath, helperPath); + var active = await service.CreateAsync( + new LinkPolicyCreateRequest( + "ai-agent", "home", "tcp", 22, 60, "integration", Guid.NewGuid().ToString()), + "windows-pc", + cancellationToken); + Assert.Equal("Active", active.ActualState); + + await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken); + var partial = await service.DisableAsync( + active.Id, + new LinkPolicyDisableRequest(Guid.NewGuid().ToString()), + "windows-pc", + cancellationToken); + Assert.NotNull(partial); + Assert.Equal("Disabled", partial.DesiredState); + Assert.Equal("Partial", partial.ActualState); + Assert.Contains("nftables validation failed", partial.LastError); + + var restartedStore = CreateStore(); + await restartedStore.InitializeAsync(cancellationToken); + var restartedService = CreateLinkService(restartedStore, sudoPath, helperPath); + var failedReconciliation = await restartedService.ReconcileDisabledLinksForNodeAsync( + "home", cancellationToken); + Assert.Equal(new LinkReconciliationResult(1, 1), failedReconciliation); + + File.Delete(failureMarkerPath); + var secondRestartStore = CreateStore(); + await secondRestartStore.InitializeAsync(cancellationToken); + var secondRestartService = CreateLinkService(secondRestartStore, sudoPath, helperPath); + var successfulReconciliation = await secondRestartService.ReconcileDisabledLinksForNodeAsync( + "home", cancellationToken); + Assert.Equal(new LinkReconciliationResult(1, 0), successfulReconciliation); + var persisted = Assert.Single(await secondRestartStore.ListEffectiveLinksForNodeAsync( + "home", cancellationToken)); + Assert.Equal("Disabled", persisted.DesiredState); + Assert.Equal("Disabled", persisted.ActualState); + + var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken); + Assert.Equal(4, invocations.Length); + Assert.Equal("link-connect ai-agent home tcp 22 60", invocations[0]); + Assert.All(invocations.Skip(1), invocation => + Assert.Equal("link-disconnect ai-agent home tcp 22", invocation)); + } + + public ValueTask DisposeAsync() + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + return ValueTask.CompletedTask; + } + + private ControlStore CreateStore() + => new(Options.Create(new ControlOptions + { + DatabasePath = Path.Combine(_directory, "control.db"), + CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx") + })); + + private static LinkService CreateLinkService( + ControlStore store, + string sudoPath, + string helperPath) + { + var applier = new LinkPolicyApplier(Options.Create(new ControlOptions + { + HubHelperPath = helperPath, + PrivilegeEscalationPath = sudoPath + })); + return new LinkService(store, applier, new ControlEventBroker()); + } + + private static async Task EnrollAgentAsync( + ControlStore store, + string nodeId, + string thumbprint, + CancellationToken cancellationToken) + { + var token = await store.CreateEnrollmentTokenAsync( + nodeId, TimeSpan.FromMinutes(10), cancellationToken); + var enrolled = await store.EnrollAsync( + new EnrollmentRequest(nodeId, token, "csr", Guid.NewGuid().ToString()), + () => new IssuedCertificate( + "certificate", "ca", thumbprint, DateTimeOffset.UtcNow.AddYears(1)), + cancellationToken); + Assert.NotNull(enrolled); + } + + [SupportedOSPlatform("linux")] + private static async Task WriteExecutableAsync( + string path, + string contents, + CancellationToken cancellationToken) + { + await File.WriteAllTextAsync( + path, + contents.Replace("\r\n", "\n"), + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + cancellationToken); + File.SetUnixFileMode( + path, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + private static string ShellQuote(string value) + => value.Replace("'", "'\\''", StringComparison.Ordinal); +}