diff --git a/docs/roadmap.md b/docs/roadmap.md index 192b6a7..11853bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -117,7 +117,7 @@ - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; - [ ] restricted root helper через Unix socket; -- [ ] structured redacted events и progress; +- [x] structured redacted events, bounded Operator history и progress; - [x] `NeedsReconciliation` после истечения execution TTL и неопределённого результата; - [ ] desired/factual configuration и drift; - [x] запрет параллельных активных заданий на одном Node (безопасный первый вариант). diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index 6bdfae8..b9f1bef 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 4; + private const int CurrentSchemaVersion = 5; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -213,6 +213,23 @@ public sealed partial class ControlStore(IOptions options) await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 5) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var addStructuredEventFields = connection.CreateCommand(); + addStructuredEventFields.Transaction = migration; + addStructuredEventFields.CommandText = """ + ALTER TABLE provisioning_events + ADD COLUMN step TEXT NOT NULL DEFAULT ''; + ALTER TABLE provisioning_events + ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0; + PRAGMA user_version = 5; + """; + await addStructuredEventFields.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( @@ -225,9 +242,10 @@ public sealed partial class ControlStore(IOptions options) var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = """ - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.expired', 'Cancelled', - 'Provisioning job expired before execution.' + 'Provisioning job expired before execution.', 'expired', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state IN ('Queued', 'AwaitingConfirmation'); @@ -238,9 +256,11 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Queued', 'AwaitingConfirmation'); SELECT changes(); - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.reconciliation.required', 'NeedsReconciliation', - 'Execution lease expired; factual state must be inspected.' + 'Execution lease expired; factual state must be inspected.', + 'reconcile', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); @@ -251,9 +271,11 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); SELECT changes(); - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation', - 'Rollback lease expired; factual state must be inspected.' + 'Rollback lease expired; factual state must be inspected.', + 'rollback-reconcile', progress_percent FROM provisioning_jobs WHERE expires_at <= $now AND state = 'RollingBack'; UPDATE provisioning_jobs SET diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index aae4368..44f0e45 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -484,6 +484,23 @@ control.MapGet("/provisioning/jobs/{id}", async ( var job = await controlStore.GetProvisioningJobAsync(id, cancellationToken); return job is null ? Results.NotFound() : Results.Ok(job); }); +control.MapGet("/provisioning/jobs/{id}/events", async ( + string id, + int? limit, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id) || limit is < 1 or > 200) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningEvents"] = ["Invalid provisioning job id or limit (1-200)."] + }); + } + var events = await controlStore.ListProvisioningEventsAsync( + id, limit ?? 100, cancellationToken); + return events is null ? Results.NotFound() : Results.Ok(events.ToArray()); +}); control.MapPost("/provisioning/jobs/{id}/confirm", async ( string id, ProvisioningJobCommandRequest request, diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 1269dc6..f6a81a8 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -248,7 +248,8 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, request.EventCode, request.State, - request.Message, now, cancellationToken); + $"Agent reported '{request.EventCode}' for step '{request.Step}' at " + + $"{request.ProgressPercent}%.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -266,6 +267,44 @@ public sealed partial class ControlStore return updated; } + public async Task?> ListProvisioningEventsAsync( + string id, + int limit, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + var exists = connection.CreateCommand(); + exists.CommandText = "SELECT EXISTS(SELECT 1 FROM provisioning_jobs WHERE id = $id);"; + exists.Parameters.AddWithValue("$id", id); + if (Convert.ToInt32(await exists.ExecuteScalarAsync(cancellationToken)) != 1) + { + return null; + } + + var result = new List(); + var command = connection.CreateCommand(); + command.CommandText = """ + SELECT sequence, job_id, recorded_at, event_type, state, + step, progress_percent, message + FROM provisioning_events + WHERE job_id = $id + ORDER BY sequence DESC + LIMIT $limit; + """; + command.Parameters.AddWithValue("$id", id); + command.Parameters.AddWithValue("$limit", Math.Clamp(limit, 1, 200)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + result.Add(new ProvisioningEvent( + reader.GetInt64(0), reader.GetString(1), DateTimeOffset.Parse(reader.GetString(2)), + reader.GetString(3), reader.GetString(4), reader.GetString(5), + reader.GetInt32(6), reader.GetString(7))); + } + result.Reverse(); + return result; + } + public async Task RetryProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -336,7 +375,7 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, "job.retry.queued", updated.State, - request.Reason, now, cancellationToken); + "Operator queued a retry after reconciliation.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -418,7 +457,7 @@ public sealed partial class ControlStore await WriteProvisioningEventAsync( connection, transaction, id, "job.rollback.queued", updated.State, - request.Reason, now, cancellationToken); + "Operator queued a rollback.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -494,7 +533,8 @@ public sealed partial class ControlStore } await WriteProvisioningEventAsync( - connection, transaction, id, eventType, targetState, request.Reason, now, cancellationToken); + connection, transaction, id, eventType, targetState, + $"Operator requested '{eventType}'.", now, cancellationToken); await WriteIdempotentAsync( connection, transaction, operationKey, requestHash, updated, SmmJsonContext.Default.ProvisioningJob, cancellationToken); @@ -569,8 +609,12 @@ public sealed partial class ControlStore var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = """ - INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) - VALUES ($job, $recorded, $event, $state, $message); + INSERT INTO provisioning_events( + job_id, recorded_at, event_type, state, message, step, progress_percent) + SELECT $job, $recorded, $event, $state, $message, + current_step, progress_percent + FROM provisioning_jobs + WHERE id = $job; """; command.Parameters.AddWithValue("$job", jobId); command.Parameters.AddWithValue("$recorded", recordedAt.ToString("O")); diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index a10a631..689a60b 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -183,6 +183,16 @@ public sealed record ProvisioningJob( string CurrentStep, string? LastError); +public sealed record ProvisioningEvent( + long Sequence, + string JobId, + DateTimeOffset RecordedAt, + string EventType, + string State, + string Step, + int ProgressPercent, + string Message); + public static class ProvisioningJobStates { public const string Queued = "Queued"; diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 9e76310..04dc7fe 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -30,4 +30,6 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningJobProgressRequest))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] +[JsonSerializable(typeof(ProvisioningEvent))] +[JsonSerializable(typeof(ProvisioningEvent[]))] public sealed partial class SmmJsonContext : JsonSerializerContext; diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 15e0139..9ec249d 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -116,6 +116,13 @@ public sealed class ControlApiTests : IAsyncDisposable Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.AwaitingConfirmation, job.State); Assert.Equal(HttpStatusCode.OK, (await client.GetAsync( $"/api/v1/control/provisioning/jobs/{job.Id}", cancellationToken)).StatusCode); + using var eventsResponse = await client.GetAsync( + $"/api/v1/control/provisioning/jobs/{job.Id}/events?limit=10", cancellationToken); + Assert.Equal(HttpStatusCode.OK, eventsResponse.StatusCode); + var events = await eventsResponse.Content.ReadFromJsonAsync( + cancellationToken); + Assert.NotNull(events); + Assert.NotEmpty(events); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 4954a16..418d8cc 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(4L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(5L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index c21ab3a..f179c32 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(4L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(5L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); @@ -113,7 +113,7 @@ public sealed class ControlStoreTests : IAsyncDisposable var preflight = new ProvisioningJobProgressRequest( ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress", - "Operating system detected.", Guid.NewGuid().ToString()); + "secret-token-should-never-be-persisted", Guid.NewGuid().ToString()); Assert.Null(await store.ReportProvisioningProgressAsync( "other", created.Id, preflight, cancellationToken)); var firstProgress = await store.ReportProvisioningProgressAsync( @@ -170,6 +170,13 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.Running, running!.State); Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State); Assert.Equal(ProvisioningJobStates.Completed, completed!.State); + var events = await store.ListProvisioningEventsAsync(created.Id, 100, cancellationToken); + Assert.NotNull(events); + var preflightEvent = Assert.Single(events!, item => item.EventType == "preflight.progress"); + Assert.Equal("inspect-os", preflightEvent.Step); + Assert.Equal(10, preflightEvent.ProgressPercent); + Assert.DoesNotContain( + events!, item => item.Message.Contains("secret-token", StringComparison.Ordinal)); } [Fact]