Persist typed provisioning preflight facts

This commit is contained in:
Ochenstarik 2026-07-19 23:14:14 +07:00
parent 4d08b93d23
commit e0c755adf1
10 changed files with 288 additions and 21 deletions

View file

@ -119,7 +119,7 @@
- [x] restricted root helper через Unix socket (первый allowlisted action `preflight`); - [x] restricted root helper через Unix socket (первый allowlisted action `preflight`);
- [x] structured redacted events, bounded Operator history и progress; - [x] structured redacted events, bounded Operator history и progress;
- [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата;
- [ ] desired/factual configuration и drift; - [ ] desired/factual configuration и drift (типизированные factual preflight facts уже сохраняются);
- [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант).
## Этап 9 — базовая настройка и пользователи ## Этап 9 — базовая настройка и пользователи

View file

@ -144,19 +144,28 @@ internal sealed class AgentClient(AgentOptions options)
return; return;
} }
var currentProgress = job.ProgressPercent; ProvisioningPreflightResult result;
try try
{ {
var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath); var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath);
var result = await helper.RunPreflightAsync(job, cancellationToken); result = await helper.RunPreflightAsync(job, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
"preflight", "preflight.failed", "Preflight helper failed.", cancellationToken);
Console.Error.WriteLine($"Preflight {job.Id} failed: {exception.Message}");
return;
}
await ReportPreflightFactsAsync(client, job, result, cancellationToken);
await ReportProvisioningAsync( await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Running, 40, client, job, ProvisioningJobStates.Running, 40,
"inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken); "inspect-host", "preflight.inspected", "Host inspection completed.", cancellationToken);
currentProgress = 40;
await ReportProvisioningAsync( await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Verifying, 80, client, job, ProvisioningJobStates.Verifying, 80,
"verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken); "verify-host", "preflight.verifying", "Verifying preflight result.", cancellationToken);
currentProgress = 80;
await ReportProvisioningAsync( await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Completed, 100, client, job, ProvisioningJobStates.Completed, 100,
"completed", "preflight.completed", "Preflight completed.", cancellationToken); "completed", "preflight.completed", "Preflight completed.", cancellationToken);
@ -164,14 +173,6 @@ internal sealed class AgentClient(AgentOptions options)
$"Preflight {job.Id} completed: {result.OperatingSystem} " $"Preflight {job.Id} completed: {result.OperatingSystem} "
+ $"{result.OperatingSystemVersion} {result.Architecture}."); + $"{result.OperatingSystemVersion} {result.Architecture}.");
} }
catch (Exception exception) when (exception is not OperationCanceledException)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Failed, currentProgress,
"preflight", "preflight.failed", "Preflight helper failed.", cancellationToken);
Console.Error.WriteLine($"Preflight {job.Id} failed: {exception.Message}");
}
}
private static async Task ReportProvisioningAsync( private static async Task ReportProvisioningAsync(
HttpClient client, HttpClient client,
@ -193,6 +194,22 @@ internal sealed class AgentClient(AgentOptions options)
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
} }
private static async Task ReportPreflightFactsAsync(
HttpClient client,
ProvisioningJob job,
ProvisioningPreflightResult facts,
CancellationToken cancellationToken)
{
var request = new ProvisioningPreflightReportRequest(
facts, DateTimeOffset.UtcNow, CreateOperationId(job.Id, "facts"));
using var response = await client.PostAsJsonAsync(
$"api/v1/agents/provisioning/jobs/{job.Id}/preflight-facts",
request,
SmmJsonContext.Default.ProvisioningPreflightReportRequest,
cancellationToken);
response.EnsureSuccessStatusCode();
}
private static string CreateOperationId(string jobId, string state) private static string CreateOperationId(string jobId, string state)
{ {
var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}")); var digest = SHA256.HashData(Encoding.UTF8.GetBytes($"{jobId}:{state}"));

View file

@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control;
public sealed partial class ControlStore(IOptions<ControlOptions> options) public sealed partial class ControlStore(IOptions<ControlOptions> options)
{ {
private const int CurrentSchemaVersion = 5; private const int CurrentSchemaVersion = 6;
private readonly ControlOptions _options = options.Value; private readonly ControlOptions _options = options.Value;
private readonly string _connectionString = new SqliteConnectionStringBuilder private readonly string _connectionString = new SqliteConnectionStringBuilder
{ {
@ -230,6 +230,27 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken); await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken);
await migration.CommitAsync(cancellationToken); await migration.CommitAsync(cancellationToken);
} }
if (schemaVersion < 6)
{
await using var migration =
(SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
var addNodePreflightFacts = connection.CreateCommand();
addNodePreflightFacts.Transaction = migration;
addNodePreflightFacts.CommandText = """
CREATE TABLE IF NOT EXISTS node_preflight_facts (
node_id TEXT PRIMARY KEY REFERENCES agents(node_id) ON DELETE CASCADE,
schema_version INTEGER NOT NULL,
facts_json TEXT NOT NULL,
observed_at TEXT NOT NULL,
source_job_id TEXT NOT NULL REFERENCES provisioning_jobs(id),
updated_at TEXT NOT NULL
);
PRAGMA user_version = 6;
""";
await addNodePreflightFacts.ExecuteNonQueryAsync(cancellationToken);
await migration.CommitAsync(cancellationToken);
}
} }
public async Task<ControlMaintenanceResult> MaintainAsync( public async Task<ControlMaintenanceResult> MaintainAsync(

View file

@ -426,10 +426,59 @@ agents.MapPost("/provisioning/jobs/{id}/progress", async (
return Results.Conflict(new ProblemDetails { Title = exception.Message }); return Results.Conflict(new ProblemDetails { Title = exception.Message });
} }
}); });
agents.MapPost("/provisioning/jobs/{id}/preflight-facts", async (
string id,
ProvisioningPreflightReportRequest request,
HttpContext context,
ControlStore controlStore,
CancellationToken cancellationToken) =>
{
var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
if (string.IsNullOrWhiteSpace(nodeId)
|| !NodeIdValidator.IsValid(nodeId)
|| !ProvisioningJobValidator.IsValidId(id)
|| !ProvisioningJobValidator.IsValid(request))
{
return Results.ValidationProblem(new Dictionary<string, string[]>
{
["preflightFacts"] = ["Invalid job id, facts, observation time, or idempotency key."]
});
}
try
{
var facts = await controlStore.RecordPreflightFactsAsync(
nodeId, id, request, cancellationToken);
return facts is null ? Results.NotFound() : Results.Ok(facts);
}
catch (IdempotencyConflictException)
{
return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
}
catch (ProvisioningTransitionException exception)
{
return Results.Conflict(new ProblemDetails { Title = exception.Message });
}
});
var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator");
control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) =>
Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray())); Results.Ok((await controlStore.ListAgentsAsync(cancellationToken)).ToArray()));
control.MapGet("/agents/{nodeId}/facts/preflight", async (
string nodeId,
ControlStore controlStore,
CancellationToken cancellationToken) =>
{
if (!NodeIdValidator.IsValid(nodeId))
{
return Results.ValidationProblem(new Dictionary<string, string[]>
{
["nodeId"] = ["Invalid node id."]
});
}
var facts = await controlStore.GetPreflightFactsAsync(nodeId, cancellationToken);
return facts is null ? Results.NotFound() : Results.Ok(facts);
});
control.MapPost("/agents/{nodeId}/provisioning/jobs", async ( control.MapPost("/agents/{nodeId}/provisioning/jobs", async (
string nodeId, string nodeId,
ProvisioningJobCreateRequest request, ProvisioningJobCreateRequest request,
@ -879,6 +928,15 @@ internal static class ProvisioningJobValidator
&& request.Message.Length <= 512 && request.Message.Length <= 512
&& IdempotencyKeyValidator.IsValid(request.IdempotencyKey); && IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
public static bool IsValid(ProvisioningPreflightReportRequest request)
=> request.ObservedAt >= DateTimeOffset.UtcNow.AddHours(-1)
&& request.ObservedAt <= DateTimeOffset.UtcNow.AddMinutes(1)
&& request.Facts is not null
&& IsSafeFact(request.Facts.OperatingSystem, 32)
&& IsSafeFact(request.Facts.OperatingSystemVersion, 64)
&& request.Facts.Architecture is "x64" or "x86" or "arm" or "arm64"
&& IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
public static bool IsValidId(string id) public static bool IsValidId(string id)
=> id.Length == 32 && Guid.TryParseExact(id, "N", out _); => id.Length == 32 && Guid.TryParseExact(id, "N", out _);
@ -890,4 +948,10 @@ internal static class ProvisioningJobValidator
&& value.All(character => character is >= 'a' and <= 'z' && value.All(character => character is >= 'a' and <= 'z'
or >= '0' and <= '9' or >= '0' and <= '9'
or '.' or '-' or '_'); or '.' or '-' or '_');
private static bool IsSafeFact(string? value, int maximumLength)
=> value is not null
&& value.Length is >= 1 && value.Length <= maximumLength
&& value.All(character => char.IsAsciiLetterOrDigit(character)
|| character is '_' or '-' or '.');
} }

View file

@ -0,0 +1,100 @@
using System.Text.Json;
using Microsoft.Data.Sqlite;
using ServerMonitorManager.Core;
namespace ServerMonitorManager.Control;
public sealed partial class ControlStore
{
public async Task<NodePreflightFacts?> RecordPreflightFactsAsync(
string nodeId,
string jobId,
ProvisioningPreflightReportRequest request,
CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken);
await using var transaction =
(SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
var operationKey = $"provisioning-preflight:{nodeId}:{jobId}:{request.IdempotencyKey}";
var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningPreflightReportRequest);
var cached = await ReadIdempotentAsync(
connection, transaction, operationKey, requestHash,
SmmJsonContext.Default.NodePreflightFacts, cancellationToken);
if (cached is not null)
{
await transaction.CommitAsync(cancellationToken);
return cached;
}
var job = await ReadProvisioningJobAsync(connection, transaction, jobId, cancellationToken);
if (job is null || !string.Equals(job.NodeId, nodeId, StringComparison.Ordinal))
{
await transaction.RollbackAsync(cancellationToken);
return null;
}
if (job.ActionType != "preflight" || job.SchemaVersion != 1
|| job.State != ProvisioningJobStates.Preflight)
{
throw new ProvisioningTransitionException(job.State, "PreflightFacts");
}
var facts = new NodePreflightFacts(
nodeId, 1, request.Facts, request.ObservedAt, jobId, DateTimeOffset.UtcNow);
var upsert = connection.CreateCommand();
upsert.Transaction = transaction;
upsert.CommandText = """
INSERT INTO node_preflight_facts(
node_id, schema_version, facts_json, observed_at, source_job_id, updated_at)
VALUES ($node, $schema, $facts, $observed, $job, $updated)
ON CONFLICT(node_id) DO UPDATE SET
schema_version = excluded.schema_version,
facts_json = excluded.facts_json,
observed_at = excluded.observed_at,
source_job_id = excluded.source_job_id,
updated_at = excluded.updated_at;
""";
upsert.Parameters.AddWithValue("$node", facts.NodeId);
upsert.Parameters.AddWithValue("$schema", facts.SchemaVersion);
upsert.Parameters.AddWithValue(
"$facts", JsonSerializer.Serialize(facts.Facts, SmmJsonContext.Default.ProvisioningPreflightResult));
upsert.Parameters.AddWithValue("$observed", facts.ObservedAt.ToString("O"));
upsert.Parameters.AddWithValue("$job", facts.SourceJobId);
upsert.Parameters.AddWithValue("$updated", facts.UpdatedAt.ToString("O"));
await upsert.ExecuteNonQueryAsync(cancellationToken);
await WriteProvisioningEventAsync(
connection, transaction, jobId, "preflight.facts-recorded", job.State,
"Validated preflight facts were recorded.", facts.UpdatedAt, cancellationToken);
await WriteIdempotentAsync(
connection, transaction, operationKey, requestHash, facts,
SmmJsonContext.Default.NodePreflightFacts, cancellationToken);
await WriteAuditAsync(
connection, transaction, nodeId, "provisioning.preflight.facts", jobId,
JsonSerializer.Serialize(new { facts.SchemaVersion, facts.ObservedAt }), cancellationToken);
await transaction.CommitAsync(cancellationToken);
return facts;
}
public async Task<NodePreflightFacts?> GetPreflightFactsAsync(
string nodeId,
CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken);
var command = connection.CreateCommand();
command.CommandText = """
SELECT schema_version, facts_json, observed_at, source_job_id, updated_at
FROM node_preflight_facts WHERE node_id = $node;
""";
command.Parameters.AddWithValue("$node", nodeId);
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
if (!await reader.ReadAsync(cancellationToken))
{
return null;
}
var facts = JsonSerializer.Deserialize(
reader.GetString(1), SmmJsonContext.Default.ProvisioningPreflightResult)
?? throw new InvalidDataException("Stored preflight facts are invalid.");
return new NodePreflightFacts(
nodeId, reader.GetInt32(0), facts, DateTimeOffset.Parse(reader.GetString(2)),
reader.GetString(3), DateTimeOffset.Parse(reader.GetString(4)));
}
}

View file

@ -179,6 +179,7 @@ public sealed record ProvisioningHelperResponse(
string Message, string Message,
ProvisioningPreflightResult? Preflight); ProvisioningPreflightResult? Preflight);
[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
public sealed record ProvisioningPreflightResult( public sealed record ProvisioningPreflightResult(
string OperatingSystem, string OperatingSystem,
string OperatingSystemVersion, string OperatingSystemVersion,
@ -189,6 +190,20 @@ public sealed record ProvisioningPreflightResult(
bool HasWireGuard, bool HasWireGuard,
bool HasApt); bool HasApt);
[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
public sealed record ProvisioningPreflightReportRequest(
ProvisioningPreflightResult Facts,
DateTimeOffset ObservedAt,
string IdempotencyKey);
public sealed record NodePreflightFacts(
string NodeId,
int SchemaVersion,
ProvisioningPreflightResult Facts,
DateTimeOffset ObservedAt,
string SourceJobId,
DateTimeOffset UpdatedAt);
public sealed record ProvisioningJob( public sealed record ProvisioningJob(
string Id, string Id,
string NodeId, string NodeId,

View file

@ -31,6 +31,8 @@ namespace ServerMonitorManager.Core;
[JsonSerializable(typeof(ProvisioningHelperRequest))] [JsonSerializable(typeof(ProvisioningHelperRequest))]
[JsonSerializable(typeof(ProvisioningHelperResponse))] [JsonSerializable(typeof(ProvisioningHelperResponse))]
[JsonSerializable(typeof(ProvisioningPreflightResult))] [JsonSerializable(typeof(ProvisioningPreflightResult))]
[JsonSerializable(typeof(ProvisioningPreflightReportRequest))]
[JsonSerializable(typeof(NodePreflightFacts))]
[JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob))]
[JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningJob[]))]
[JsonSerializable(typeof(ProvisioningEvent))] [JsonSerializable(typeof(ProvisioningEvent))]

View file

@ -168,6 +168,54 @@ public sealed class ControlApiTests : IAsyncDisposable
Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync( Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync(
"/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);
var preflightReport = new
{
facts = new
{
operatingSystem = "ubuntu",
operatingSystemVersion = "24.04",
architecture = "x64",
hasSystemd = true,
hasSshd = true,
hasNftables = true,
hasWireGuard = false,
hasApt = true
},
observedAt = DateTimeOffset.UtcNow,
idempotencyKey = Guid.NewGuid().ToString()
};
Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync(
$"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
preflightReport,
cancellationToken)).StatusCode);
using var factsResponse = await assigned.PostAsJsonAsync(
$"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
preflightReport,
cancellationToken);
Assert.Equal(HttpStatusCode.OK, factsResponse.StatusCode);
var recorded = await factsResponse.Content.ReadFromJsonAsync<ServerMonitorManager.Core.NodePreflightFacts>(
cancellationToken);
Assert.Equal(nodeId, recorded!.NodeId);
Assert.Equal("ubuntu", recorded.Facts.OperatingSystem);
using var replayResponse = await assigned.PostAsJsonAsync(
$"/api/v1/agents/provisioning/jobs/{created.Id}/preflight-facts",
preflightReport,
cancellationToken);
Assert.Equal(HttpStatusCode.OK, replayResponse.StatusCode);
var replayed = await replayResponse.Content
.ReadFromJsonAsync<ServerMonitorManager.Core.NodePreflightFacts>(cancellationToken);
Assert.Equal(recorded.UpdatedAt, replayed!.UpdatedAt);
using var operatorClient = _factory.CreateClient();
operatorClient.DefaultRequestHeaders.Add("X-Test-Identity", "windows-pc");
operatorClient.DefaultRequestHeaders.Add("X-Test-Role", "Operator");
using var storedFactsResponse = await operatorClient.GetAsync(
$"/api/v1/control/agents/{nodeId}/facts/preflight", cancellationToken);
Assert.Equal(HttpStatusCode.OK, storedFactsResponse.StatusCode);
var stored = await storedFactsResponse.Content
.ReadFromJsonAsync<ServerMonitorManager.Core.NodePreflightFacts>(cancellationToken);
Assert.Equal(created.Id, stored!.SourceJobId);
var progress = new var progress = new
{ {
state = "Running", state = "Running",

View file

@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
await verify.OpenAsync(cancellationToken); await verify.OpenAsync(cancellationToken);
var version = verify.CreateCommand(); var version = verify.CreateCommand();
version.CommandText = "PRAGMA user_version;"; version.CommandText = "PRAGMA user_version;";
Assert.Equal(5L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); Assert.Equal(6L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
} }
[Fact] [Fact]

View file

@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable
await migrated.OpenAsync(TestContext.Current.CancellationToken); await migrated.OpenAsync(TestContext.Current.CancellationToken);
var version = migrated.CreateCommand(); var version = migrated.CreateCommand();
version.CommandText = "PRAGMA user_version;"; version.CommandText = "PRAGMA user_version;";
Assert.Equal(5L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); Assert.Equal(6L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
var table = migrated.CreateCommand(); var table = migrated.CreateCommand();
table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');";
Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken)));