241 lines
11 KiB
C#
241 lines
11 KiB
C#
using Microsoft.Data.Sqlite;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using System.Text.Json;
|
|
using ServerMonitorManager.Control;
|
|
using ServerMonitorManager.Core;
|
|
using Xunit;
|
|
|
|
namespace ServerMonitorManager.Control.Tests;
|
|
|
|
public sealed class ControlMaintenanceTests : IAsyncDisposable
|
|
{
|
|
private readonly string _directory = Path.Combine(
|
|
Path.GetTempPath(), $"smm-maintenance-tests-{Guid.NewGuid():N}");
|
|
|
|
[Fact]
|
|
public async Task ExpiredLinkIsDisabledAndFirewallStateIsReconciled()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var (store, _) = CreateServices();
|
|
await store.InitializeAsync(cancellationToken);
|
|
await EnrollAgentAsync(store, "ai-agent", "AA11", cancellationToken);
|
|
await EnrollAgentAsync(store, "home", "BB22", cancellationToken);
|
|
var applier = new RecordingPolicyApplier();
|
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
|
var link = await service.CreateAsync(
|
|
new LinkPolicyCreateRequest(
|
|
"ai-agent", "home", "tcp", 22, 1, "ttl-test", Guid.NewGuid().ToString()),
|
|
"windows-pc",
|
|
cancellationToken);
|
|
|
|
var result = await service.ExpireDueLinksAsync(
|
|
link.CreatedAt.AddMinutes(2), cancellationToken);
|
|
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
|
|
|
Assert.Equal(new LinkExpirationResult(1, 0), result);
|
|
Assert.Equal("Disabled", persisted.DesiredState);
|
|
Assert.Equal("Disabled", persisted.ActualState);
|
|
Assert.Equal(1, applier.DisconnectCalls);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ExpirationRetriesPartialFirewallRemoval()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var (store, _) = CreateServices();
|
|
await store.InitializeAsync(cancellationToken);
|
|
await EnrollAgentAsync(store, "ai-agent", "CC33", cancellationToken);
|
|
await EnrollAgentAsync(store, "home", "DD44", cancellationToken);
|
|
var applier = new RecordingPolicyApplier { FailDisconnect = true };
|
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
|
var link = await service.CreateAsync(
|
|
new LinkPolicyCreateRequest(
|
|
"ai-agent", "home", "tcp", 22, 1, "retry-test", Guid.NewGuid().ToString()),
|
|
"windows-pc",
|
|
cancellationToken);
|
|
var expiredAt = link.CreatedAt.AddMinutes(2);
|
|
|
|
Assert.Equal(new LinkExpirationResult(0, 1),
|
|
await service.ExpireDueLinksAsync(expiredAt, cancellationToken));
|
|
applier.FailDisconnect = false;
|
|
Assert.Equal(new LinkExpirationResult(1, 0),
|
|
await service.ExpireDueLinksAsync(expiredAt.AddSeconds(1), cancellationToken));
|
|
|
|
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
|
Assert.Equal("Disabled", persisted.ActualState);
|
|
Assert.Equal(2, applier.DisconnectCalls);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MaintenancePrunesExpiredOperationalDataAndVersionsSchema()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var (store, options) = CreateServices();
|
|
await store.InitializeAsync(cancellationToken);
|
|
await EnrollAgentAsync(store, "home", "EE55", cancellationToken);
|
|
var now = DateTimeOffset.UtcNow;
|
|
await store.RecordHeartbeatAsync(
|
|
new AgentHeartbeat(
|
|
"home", "test", now, 0.2, 1, 2, 1, 2, 3, 4, 5, Guid.NewGuid().ToString()),
|
|
30,
|
|
cancellationToken);
|
|
|
|
await using (var connection = new SqliteConnection($"Data Source={options.DatabasePath}"))
|
|
{
|
|
await connection.OpenAsync(cancellationToken);
|
|
var age = connection.CreateCommand();
|
|
age.CommandText = """
|
|
UPDATE metric_samples SET recorded_at = $old;
|
|
UPDATE idempotency SET created_at = $old;
|
|
UPDATE audit SET recorded_at = $old;
|
|
INSERT INTO device_tokens(token_hash, device_id, expires_at, consumed_at)
|
|
VALUES ('expired', 'old-device', $old, NULL);
|
|
""";
|
|
age.Parameters.AddWithValue("$old", now.AddDays(-400).ToString("O"));
|
|
await age.ExecuteNonQueryAsync(cancellationToken);
|
|
}
|
|
|
|
var result = await store.MaintainAsync(now, cancellationToken);
|
|
Assert.True(result.MetricsDeleted >= 1);
|
|
Assert.True(result.IdempotencyDeleted >= 1);
|
|
Assert.True(result.AuditDeleted >= 1);
|
|
Assert.True(result.TokensDeleted >= 1);
|
|
|
|
await using var verify = new SqliteConnection($"Data Source={options.DatabasePath}");
|
|
await verify.OpenAsync(cancellationToken);
|
|
var version = verify.CreateCommand();
|
|
version.CommandText = "PRAGMA user_version;";
|
|
Assert.Equal(8L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ExpiredProvisioningJobsAreCancelledOrRequireReconciliationAndCanRetry()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var (store, _) = CreateServices();
|
|
await store.InitializeAsync(cancellationToken);
|
|
await EnrollAgentAsync(store, "queued-node", "AABB", cancellationToken);
|
|
await EnrollAgentAsync(store, "running-node", "CCDD", cancellationToken);
|
|
using var parameters = JsonDocument.Parse("{}");
|
|
var queued = await store.CreateProvisioningJobAsync(
|
|
"queued-node",
|
|
new ProvisioningJobCreateRequest(
|
|
"system.base-install", 1, parameters.RootElement.Clone(), 5,
|
|
"Await approval", Guid.NewGuid().ToString()),
|
|
"operator",
|
|
cancellationToken);
|
|
var running = await store.CreateProvisioningJobAsync(
|
|
"running-node",
|
|
new ProvisioningJobCreateRequest(
|
|
"preflight", 1, parameters.RootElement.Clone(), 5,
|
|
"Inspect node", Guid.NewGuid().ToString()),
|
|
"operator",
|
|
cancellationToken);
|
|
Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken));
|
|
|
|
var result = await store.MaintainAsync(
|
|
running.ExpiresAt.AddSeconds(1), cancellationToken);
|
|
|
|
Assert.Equal(1, result.ProvisioningJobsCancelled);
|
|
Assert.Equal(1, result.ProvisioningJobsNeedingReconciliation);
|
|
Assert.Equal(ProvisioningJobStates.Cancelled,
|
|
(await store.GetProvisioningJobAsync(queued.Id, cancellationToken))!.State);
|
|
var reconciliation = await store.GetProvisioningJobAsync(running.Id, cancellationToken);
|
|
Assert.Equal(ProvisioningJobStates.NeedsReconciliation, reconciliation!.State);
|
|
Assert.Equal("job.ttl_expired", reconciliation.LastError);
|
|
|
|
var retried = await store.RetryProvisioningJobAsync(
|
|
running.Id,
|
|
new ProvisioningJobCommandRequest("Factual state checked", Guid.NewGuid().ToString()),
|
|
"operator",
|
|
cancellationToken);
|
|
Assert.Equal(ProvisioningJobStates.Queued, retried!.State);
|
|
Assert.Equal(0, retried.ProgressPercent);
|
|
Assert.Null(retried.LastError);
|
|
Assert.True(retried.ExpiresAt > running.ExpiresAt);
|
|
Assert.NotNull(await store.ClaimNextProvisioningJobAsync("running-node", cancellationToken));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task BackupCanRestoreDatabaseAndCertificateAuthority()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var (store, options) = CreateServices();
|
|
await File.WriteAllBytesAsync(options.CertificateAuthorityPath, [1, 2, 3, 4], cancellationToken);
|
|
await store.InitializeAsync(cancellationToken);
|
|
await EnrollAgentAsync(store, "home", "FF66", cancellationToken);
|
|
var backups = new ControlBackupService(
|
|
store, Options.Create(options), NullLogger<ControlBackupService>.Instance);
|
|
var backupPath = await backups.CreateAsync(DateTimeOffset.UtcNow, cancellationToken);
|
|
|
|
SqliteConnection.ClearAllPools();
|
|
File.Delete(options.DatabasePath);
|
|
await File.WriteAllBytesAsync(options.CertificateAuthorityPath, [9, 9], cancellationToken);
|
|
await backups.RestoreAsync(backupPath, cancellationToken);
|
|
|
|
var restoredStore = new ControlStore(Options.Create(options));
|
|
await restoredStore.InitializeAsync(cancellationToken);
|
|
Assert.Single(await restoredStore.ListAgentsAsync(cancellationToken));
|
|
Assert.Equal([1, 2, 3, 4], await File.ReadAllBytesAsync(
|
|
options.CertificateAuthorityPath, cancellationToken));
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
SqliteConnection.ClearAllPools();
|
|
if (Directory.Exists(_directory))
|
|
{
|
|
Directory.Delete(_directory, recursive: true);
|
|
}
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
|
|
private (ControlStore Store, ControlOptions Options) CreateServices()
|
|
{
|
|
Directory.CreateDirectory(_directory);
|
|
var options = new ControlOptions
|
|
{
|
|
DatabasePath = Path.Combine(_directory, "control.db"),
|
|
CertificateAuthorityPath = Path.Combine(_directory, "control-ca.pfx"),
|
|
BackupDirectory = Path.Combine(_directory, "backups"),
|
|
MetricRetentionHours = 24,
|
|
IdempotencyRetentionHours = 1,
|
|
AuditRetentionDays = 1
|
|
};
|
|
return (new ControlStore(Options.Create(options)), options);
|
|
}
|
|
|
|
private static async Task EnrollAgentAsync(
|
|
ControlStore store,
|
|
string nodeId,
|
|
string thumbprint,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var token = await store.CreateEnrollmentTokenAsync(
|
|
nodeId, TimeSpan.FromMinutes(10), cancellationToken);
|
|
var result = await store.EnrollAsync(
|
|
new EnrollmentRequest(nodeId, token, "csr", Guid.NewGuid().ToString()),
|
|
() => new IssuedCertificate(
|
|
"certificate", "ca", thumbprint, DateTimeOffset.UtcNow.AddYears(1)),
|
|
cancellationToken);
|
|
Assert.NotNull(result);
|
|
}
|
|
|
|
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
|
{
|
|
public bool FailDisconnect { get; set; }
|
|
public int DisconnectCalls { get; private set; }
|
|
|
|
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
|
=> Task.CompletedTask;
|
|
|
|
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
|
{
|
|
DisconnectCalls++;
|
|
return FailDisconnect
|
|
? Task.FromException(new InvalidOperationException("simulated firewall failure"))
|
|
: Task.CompletedTask;
|
|
}
|
|
}
|
|
}
|