From 722f08455dbc79f51654e073b9be1efc3af2cf1b Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 21:35:30 +0700 Subject: [PATCH] Add provisioning progress state machine --- .../ControlStore.cs | 19 ++- src/ServerMonitorManager.Control/Program.cs | 54 ++++++++ .../ProvisioningStore.cs | 122 +++++++++++++++++- src/ServerMonitorManager.Core/Contracts.cs | 15 +++ .../SmmJsonContext.cs | 1 + .../ControlApiTests.cs | 23 ++++ .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 64 ++++++++- 8 files changed, 295 insertions(+), 5 deletions(-) diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index caef29c..bcb5370 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 2; + private const int CurrentSchemaVersion = 3; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -179,6 +179,23 @@ public sealed partial class ControlStore(IOptions options) await migrateProvisioning.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 3) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var migrateProgress = connection.CreateCommand(); + migrateProgress.Transaction = migration; + migrateProgress.CommandText = """ + ALTER TABLE provisioning_jobs + ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0; + ALTER TABLE provisioning_jobs + ADD COLUMN current_step TEXT NOT NULL DEFAULT ''; + PRAGMA user_version = 3; + """; + await migrateProgress.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index e84c0fe..31a6ed5 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -391,6 +391,41 @@ agents.MapGet("/provisioning/jobs/next", async ( var job = await controlStore.ClaimNextProvisioningJobAsync(nodeId, cancellationToken); return job is null ? Results.NoContent() : Results.Ok(job); }); +agents.MapPost("/provisioning/jobs/{id}/progress", async ( + string id, + ProvisioningJobProgressRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier); + if (string.IsNullOrWhiteSpace(nodeId) + || !NodeIdValidator.IsValid(nodeId) + || !ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningProgress"] = + ["Invalid job id, state, progress, step, event code, message, or idempotency key."] + }); + } + + try + { + var job = await controlStore.ReportProvisioningProgressAsync( + nodeId, id, request, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator"); control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) => @@ -749,9 +784,28 @@ internal static class ProvisioningJobValidator => request.Reason.Length is >= 1 and <= 256 && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValid(ProvisioningJobProgressRequest request) + => request.State is ProvisioningJobStates.Preflight + or ProvisioningJobStates.Running + or ProvisioningJobStates.Verifying + or ProvisioningJobStates.Completed + or ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + && request.ProgressPercent is >= 0 and <= 100 + && IsSafeCode(request.Step, 64) + && IsSafeCode(request.EventCode, 64) + && request.Message.Length <= 512 + && IdempotencyKeyValidator.IsValid(request.IdempotencyKey); + public static bool IsValidId(string id) => id.Length == 32 && Guid.TryParseExact(id, "N", out _); public static bool RequiresConfirmation(string actionType) => actionType == "system.base-install"; + + private static bool IsSafeCode(string value, int maximumLength) + => value.Length is >= 1 && value.Length <= maximumLength + && value.All(character => character is >= 'a' and <= 'z' + or >= '0' and <= '9' + or '.' or '-' or '_'); } diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 0c20ad2..35c7484 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -54,6 +54,8 @@ public sealed partial class ControlStore null, null, 1, + 0, + "queued", null); var insert = connection.CreateCommand(); @@ -166,6 +168,96 @@ public sealed partial class ControlStore "job.cancelled", "provisioning.job.cancelled", setConfirmedAt: false, cancellationToken); + public async Task ReportProvisioningProgressAsync( + string nodeId, + string id, + ProvisioningJobProgressRequest request, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-progress:{nodeId}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobProgressRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null || !string.Equals(current.NodeId, nodeId, StringComparison.Ordinal)) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (!ProvisioningStateMachine.CanReport( + current.State, request.State, current.ProgressPercent, request.ProgressPercent)) + { + throw new ProvisioningTransitionException(current.State, request.State); + } + + var now = DateTimeOffset.UtcNow; + var updated = current with + { + State = request.State, + ProgressPercent = request.ProgressPercent, + CurrentStep = request.Step, + UpdatedAt = now, + Version = current.Version + 1, + LastError = request.State is ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + ? request.EventCode + : null + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, + progress_percent = $progress, + current_step = $step, + updated_at = $updated, + version = $version, + last_error = $error + WHERE id = $id AND node_id = $node AND version = $previous_version; + """; + update.Parameters.AddWithValue("$state", updated.State); + update.Parameters.AddWithValue("$progress", updated.ProgressPercent); + update.Parameters.AddWithValue("$step", updated.CurrentStep); + update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O")); + update.Parameters.AddWithValue("$version", updated.Version); + update.Parameters.AddWithValue("$error", (object?)updated.LastError ?? DBNull.Value); + update.Parameters.AddWithValue("$id", updated.Id); + update.Parameters.AddWithValue("$node", nodeId); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, request.State); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, request.EventCode, request.State, + request.Message, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, nodeId, "provisioning.job.progress", id, + JsonSerializer.Serialize(new + { + request.State, + request.ProgressPercent, + request.Step, + request.EventCode + }), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + private async Task TransitionProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -265,7 +357,8 @@ public sealed partial class ControlStore DateTimeOffset.Parse(reader.GetString(11)), reader.IsDBNull(12) ? null : DateTimeOffset.Parse(reader.GetString(12)), reader.IsDBNull(13) ? null : DateTimeOffset.Parse(reader.GetString(13)), - reader.GetInt64(14), reader.IsDBNull(15) ? null : reader.GetString(15)); + reader.GetInt64(14), reader.GetInt32(16), reader.GetString(17), + reader.IsDBNull(15) ? null : reader.GetString(15)); private static JsonElement ParseParameters(string json) { @@ -321,3 +414,30 @@ public sealed class ProvisioningNodeNotFoundException(string nodeId) : Exception public sealed class ProvisioningTransitionException(string state, string targetState) : Exception($"Provisioning job cannot transition from '{state}' to '{targetState}'."); + +internal static class ProvisioningStateMachine +{ + public static bool CanReport(string current, string target, int currentProgress, int targetProgress) + { + if (targetProgress < currentProgress || targetProgress is < 0 or > 100) + { + return false; + } + if (target is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation) + { + return current is ProvisioningJobStates.Preflight + or ProvisioningJobStates.Running + or ProvisioningJobStates.Verifying; + } + return (current, target) switch + { + (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true, + (ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true, + (ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true, + (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100, + _ => false + }; + } +} diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index 8622801..dda378e 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -155,6 +155,14 @@ public sealed record ProvisioningJobCommandRequest( string Reason, string IdempotencyKey); +public sealed record ProvisioningJobProgressRequest( + string State, + int ProgressPercent, + string Step, + string EventCode, + string Message, + string IdempotencyKey); + public sealed record ProvisioningJob( string Id, string NodeId, @@ -171,6 +179,8 @@ public sealed record ProvisioningJob( DateTimeOffset? ConfirmedAt, DateTimeOffset? CancelledAt, long Version, + int ProgressPercent, + string CurrentStep, string? LastError); public static class ProvisioningJobStates @@ -178,5 +188,10 @@ public static class ProvisioningJobStates public const string Queued = "Queued"; public const string Preflight = "Preflight"; public const string AwaitingConfirmation = "AwaitingConfirmation"; + public const string Running = "Running"; + public const string Verifying = "Verifying"; + public const string Completed = "Completed"; + public const string Failed = "Failed"; + public const string NeedsReconciliation = "NeedsReconciliation"; public const string Cancelled = "Cancelled"; } diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index d58cb45..9e76310 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -27,6 +27,7 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ControlError))] [JsonSerializable(typeof(ProvisioningJobCreateRequest))] [JsonSerializable(typeof(ProvisioningJobCommandRequest))] +[JsonSerializable(typeof(ProvisioningJobProgressRequest))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] public sealed partial class SmmJsonContext : JsonSerializerContext; diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 17a0749..15e0139 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -160,6 +160,29 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State); Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync( "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); + + var progress = new + { + state = "Running", + progressPercent = 25, + step = "apply", + eventCode = "apply.started", + message = "Applying the approved plan.", + idempotencyKey = Guid.NewGuid().ToString() + }; + Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/progress", + progress, + cancellationToken)).StatusCode); + using var progressResponse = await assigned.PostAsJsonAsync( + $"/api/v1/agents/provisioning/jobs/{created.Id}/progress", + progress, + cancellationToken); + Assert.Equal(HttpStatusCode.OK, progressResponse.StatusCode); + var updated = await progressResponse.Content.ReadFromJsonAsync( + cancellationToken); + Assert.Equal(25, updated!.ProgressPercent); + Assert.Equal("apply", updated.CurrentStep); } public async ValueTask DisposeAsync() => await _factory.DisposeAsync(); diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index adcee40..f70a384 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -105,7 +105,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(2L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index 87bba8c..40be431 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,10 +31,10 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(2L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; - Assert.Equal(16L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); } [Fact] @@ -110,6 +110,66 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal("home", claimed.NodeId); Assert.Equal(ProvisioningJobStates.Preflight, claimed.State); Assert.Null(Assert.Single(claims, job => job is null)); + + var preflight = new ProvisioningJobProgressRequest( + ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress", + "Operating system detected.", Guid.NewGuid().ToString()); + Assert.Null(await store.ReportProvisioningProgressAsync( + "other", created.Id, preflight, cancellationToken)); + var firstProgress = await store.ReportProvisioningProgressAsync( + "home", created.Id, preflight, cancellationToken); + var replay = await store.ReportProvisioningProgressAsync( + "home", created.Id, preflight, cancellationToken); + Assert.Equal(firstProgress!.Id, replay!.Id); + Assert.Equal(10, replay.ProgressPercent); + Assert.Equal("inspect-os", replay.CurrentStep); + await Assert.ThrowsAsync(() => + store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Completed, + ProgressPercent = 100, + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken)); + + var running = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Running, + ProgressPercent = 40, + Step = "apply", + EventCode = "apply.started", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + var verifying = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Verifying, + ProgressPercent = 90, + Step = "verify", + EventCode = "verify.started", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + var completed = await store.ReportProvisioningProgressAsync( + "home", created.Id, + preflight with + { + State = ProvisioningJobStates.Completed, + ProgressPercent = 100, + Step = "complete", + EventCode = "job.completed", + IdempotencyKey = Guid.NewGuid().ToString() + }, + cancellationToken); + Assert.Equal(ProvisioningJobStates.Running, running!.State); + Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State); + Assert.Equal(ProvisioningJobStates.Completed, completed!.State); } [Fact]