diff --git a/README.md b/README.md index a2ec2ec..51c7fd8 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ In the application, generate or copy the monitoring SSH key, add the Hub profile The current development branch implements Windows SSH monitoring, the Hub/Node WireGuard installer, directional Links, one-time enrollment, separate mTLS Agent, Operator, and source-scoped Automation identities, certificate revocation/re-enrollment, SQLite control state, audit, authenticated event streaming, Windows Control API integration, and a bounded durable Agent buffer with downsampling. -Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Linux CI exercises the real Control-to-helper process boundary, including a helper failure and Control process reconstruction over the same SQLite database. Still planned: end-to-end nftables and host-reboot tests with the installer, a 50–100 Node load test, signed Windows installer, and desktop/mobile clients for additional platforms. +Reconnect reconciliation is implemented with a durable SQLite marker: after a Node returns, the Hub reapplies the latest effective disabled policies and clears the marker only after the firewall confirms success. Linux CI exercises the real Control-to-helper process boundary, including a helper failure and Control process reconstruction over the same SQLite database. CI also runs a 100-Node concurrent heartbeat and replay scenario against one Hub store. Still planned: end-to-end nftables and host-reboot tests with the installer, a signed Windows installer, and desktop/mobile clients for additional platforms. ## License and project policy diff --git a/docs/architecture.md b/docs/architecture.md index c2d7302..ebcb206 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -107,7 +107,7 @@ Agent использует ограниченный долговечный бу - без Kubernetes и обязательного Docker; - отсутствие публичного agent API; - вторичные серверы работают без белого IP; -- 50–100 узлов на одном небольшом Hub; +- 50–100 узлов на одном небольшом Hub; Control/SQLite путь проверяется CI-сценарием со 100 одновременно активными Node; - Node agent: idle RAM до 50 МБ и CPU менее 1%; - команды не повторяются без idempotency key; - потеря истории метрик не должна приводить к потере управления Links. diff --git a/docs/load-testing.md b/docs/load-testing.md new file mode 100644 index 0000000..7dfa6df --- /dev/null +++ b/docs/load-testing.md @@ -0,0 +1,26 @@ +# Нагрузочный тест Hub + +Цель текущего теста — постоянно проверять, что один Control Hub корректно принимает штатную нагрузку от 100 Node и сохраняет семантику idempotency. Это регрессионный CI-тест, а не синтетический рейтинг производительности конкретного процессора. + +## Сценарий + +`HubLoadTests.OneHubAcceptsConcurrentHeartbeatsFromOneHundredNodes` выполняет следующие действия: + +1. создаёт отдельную SQLite-базу Hub; +2. регистрирует 100 Node с разными сертификатами; +3. отправляет от всех Node три одновременные волны heartbeat; +4. ограничивает каждую волну штатным интервалом Agent в 30 секунд; +5. повторяет последнюю волну с теми же idempotency key; +6. проверяет 100 online-узлов, ровно 300 metric samples и неизменные sequence при replay. + +Таким образом CI проверяет худший случай синхронного всплеска вместо распределения heartbeat по 30-секундному окну. + +## Запуск + +```bash +dotnet test tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj \ + --configuration Release \ + --filter "Category=Load" +``` + +Тест охватывает конкурентный путь Control/SQLite. Реальные WireGuard, nftables, пропускная способность сети, TLS handshake и перезагрузка хоста относятся к отдельному инфраструктурному end-to-end тесту установщика и не имитируются этим сценарием. diff --git a/docs/roadmap.md b/docs/roadmap.md index 4e77def..fd76c21 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -77,7 +77,7 @@ - [x] защищённый Hub event stream для desktop client; - [x] ограниченный локальный буфер и downsampling; - [x] idempotency key и защита от replay; -- [ ] тест нагрузки 50–100 Node на одном Hub. +- [x] тест нагрузки 100 Node на одном Hub (конкурентные heartbeat, inventory и replay в CI). ## Этап 7 — релиз и другие платформы diff --git a/tests/ServerMonitorManager.Control.Tests/HubLoadTests.cs b/tests/ServerMonitorManager.Control.Tests/HubLoadTests.cs new file mode 100644 index 0000000..b7dd439 --- /dev/null +++ b/tests/ServerMonitorManager.Control.Tests/HubLoadTests.cs @@ -0,0 +1,141 @@ +using System.Diagnostics; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Options; +using ServerMonitorManager.Control; +using ServerMonitorManager.Core; +using Xunit; + +namespace ServerMonitorManager.Control.Tests; + +public sealed class HubLoadTests : IAsyncDisposable +{ + private const int NodeCount = 100; + private const int HeartbeatWaves = 3; + private static readonly TimeSpan WaveDeadline = TimeSpan.FromSeconds(30); + private readonly string _directory = Path.Combine( + Path.GetTempPath(), + $"smm-load-tests-{Guid.NewGuid():N}"); + + [Fact] + [Trait("Category", "Load")] + public async Task OneHubAcceptsConcurrentHeartbeatsFromOneHundredNodes() + { + var testCancellationToken = TestContext.Current.CancellationToken; + var store = CreateStore(); + await store.InitializeAsync(testCancellationToken); + await EnrollNodesAsync(store, testCancellationToken); + + var sentAt = DateTimeOffset.UtcNow.AddMinutes(-HeartbeatWaves); + var latestResponses = new AgentHeartbeatResponse[NodeCount]; + var stopwatch = Stopwatch.StartNew(); + + for (var wave = 0; wave < HeartbeatWaves; wave++) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(testCancellationToken); + deadline.CancelAfter(WaveDeadline); + var currentWave = wave; + var responses = await Task.WhenAll(Enumerable.Range(0, NodeCount).Select(async nodeIndex => + { + var mutation = await store.RecordHeartbeatAsync( + CreateHeartbeat(nodeIndex, currentWave, sentAt), + (int)WaveDeadline.TotalSeconds, + deadline.Token); + Assert.True(mutation.RequiresReconciliation); + return mutation.Response; + })); + + Assert.Equal(NodeCount, responses.Select(response => response.Sequence).Distinct().Count()); + latestResponses = responses; + } + + stopwatch.Stop(); + Assert.True( + stopwatch.Elapsed < WaveDeadline * HeartbeatWaves, + $"The heartbeat workload took {stopwatch.Elapsed}; expected less than {WaveDeadline * HeartbeatWaves}."); + + var replayResponses = await Task.WhenAll(Enumerable.Range(0, NodeCount).Select(async nodeIndex => + (await store.RecordHeartbeatAsync( + CreateHeartbeat(nodeIndex, HeartbeatWaves - 1, sentAt), + (int)WaveDeadline.TotalSeconds, + testCancellationToken)).Response)); + Assert.Equal( + latestResponses.Select(response => response.Sequence), + replayResponses.Select(response => response.Sequence)); + + var agents = await store.ListAgentsAsync(testCancellationToken); + Assert.Equal(NodeCount, agents.Count); + Assert.All(agents, agent => + { + Assert.Equal("Online", agent.Status); + Assert.Equal("load-test", agent.AgentVersion); + Assert.NotNull(agent.LastSeenAt); + }); + + await using var connection = new SqliteConnection( + $"Data Source={Path.Combine(_directory, "control.db")}"); + await connection.OpenAsync(testCancellationToken); + var metricCount = connection.CreateCommand(); + metricCount.CommandText = "SELECT COUNT(*) FROM metric_samples;"; + Assert.Equal( + NodeCount * HeartbeatWaves, + Convert.ToInt32(await metricCount.ExecuteScalarAsync(testCancellationToken))); + } + + public ValueTask DisposeAsync() + { + SqliteConnection.ClearAllPools(); + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + return ValueTask.CompletedTask; + } + + private ControlStore CreateStore() + { + Directory.CreateDirectory(_directory); + return new ControlStore(Options.Create(new ControlOptions + { + DatabasePath = Path.Combine(_directory, "control.db"), + CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx") + })); + } + + private static async Task EnrollNodesAsync(ControlStore store, CancellationToken cancellationToken) + { + for (var index = 0; index < NodeCount; index++) + { + var nodeId = NodeId(index); + var token = await store.CreateEnrollmentTokenAsync( + nodeId, + TimeSpan.FromMinutes(10), + cancellationToken); + var result = await store.EnrollAsync( + new EnrollmentRequest(nodeId, token, "csr", $"enroll-{nodeId}"), + () => new IssuedCertificate( + "certificate", + "ca", + $"LOAD{index:D3}", + DateTimeOffset.UtcNow.AddYears(1)), + cancellationToken); + Assert.NotNull(result); + } + } + + private static AgentHeartbeat CreateHeartbeat(int nodeIndex, int wave, DateTimeOffset sentAt) + => new( + NodeId(nodeIndex), + "load-test", + sentAt.AddSeconds(wave * WaveDeadline.TotalSeconds), + 0.25 + (nodeIndex % 10 / 20d), + 512L * 1024 * 1024, + 2L * 1024 * 1024 * 1024, + 8L * 1024 * 1024 * 1024, + 32L * 1024 * 1024 * 1024, + wave * 4096L, + wave * 2048L, + 3600 + wave * 30L, + $"heartbeat-{wave:D2}-{nodeIndex:D3}"); + + private static string NodeId(int index) => $"load-node-{index:D3}"; +}