From 3ae9f4717b9337a76950f10c73e9088c9ad6451f Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Sun, 19 Jul 2026 22:10:42 +0700 Subject: [PATCH] Add provisioning rollback workflow --- docs/roadmap.md | 2 +- .../ControlStore.cs | 36 +++++- src/ServerMonitorManager.Control/Program.cs | 34 ++++++ .../ProvisioningStore.cs | 107 +++++++++++++++++- src/ServerMonitorManager.Core/Contracts.cs | 3 + .../ControlMaintenanceTests.cs | 2 +- .../ControlStoreTests.cs | 90 ++++++++++++++- 7 files changed, 262 insertions(+), 12 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index bbd4e30..192b6a7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -109,7 +109,7 @@ ## Этап 8 — Provisioning control plane - [x] модели и SQLite migration v2 для ProvisioningJob; -- [ ] state machine, confirmations, cancellation, retry и rollback; +- [x] state machine, confirmations, cancellation, retry и rollback; - [x] создание, чтение, подтверждение и отмена через Operator API; - [ ] выполнение, retry, verification и rollback в полной state machine; - [x] обязательные idempotency key, audit reason и job TTL; diff --git a/src/ServerMonitorManager.Control/ControlStore.cs b/src/ServerMonitorManager.Control/ControlStore.cs index 478f698..6bdfae8 100644 --- a/src/ServerMonitorManager.Control/ControlStore.cs +++ b/src/ServerMonitorManager.Control/ControlStore.cs @@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control; public sealed partial class ControlStore(IOptions options) { - private const int CurrentSchemaVersion = 3; + private const int CurrentSchemaVersion = 4; private readonly ControlOptions _options = options.Value; private readonly string _connectionString = new SqliteConnectionStringBuilder { @@ -196,6 +196,23 @@ public sealed partial class ControlStore(IOptions options) await migrateProgress.ExecuteNonQueryAsync(cancellationToken); await migration.CommitAsync(cancellationToken); } + + if (schemaVersion < 4) + { + await using var migration = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var strengthenProvisioningLock = connection.CreateCommand(); + strengthenProvisioningLock.Transaction = migration; + strengthenProvisioningLock.CommandText = """ + DROP INDEX IF EXISTS ux_provisioning_jobs_active_node; + CREATE UNIQUE INDEX ux_provisioning_jobs_active_node + ON provisioning_jobs(node_id) + WHERE state NOT IN ('Completed', 'Cancelled', 'RolledBack'); + PRAGMA user_version = 4; + """; + await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken); + await migration.CommitAsync(cancellationToken); + } } public async Task MaintainAsync( @@ -234,6 +251,17 @@ public sealed partial class ControlStore(IOptions options) WHERE expires_at <= $now AND state IN ('Preflight', 'Running', 'Verifying'); SELECT changes(); + INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message) + SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation', + 'Rollback lease expired; factual state must be inspected.' + FROM provisioning_jobs + WHERE expires_at <= $now AND state = 'RollingBack'; + UPDATE provisioning_jobs SET + state = 'NeedsReconciliation', updated_at = $now, + current_step = 'rollback-reconcile', version = version + 1, + last_error = 'job.rollback.ttl_expired' + WHERE expires_at <= $now AND state = 'RollingBack'; + SELECT changes(); DELETE FROM metric_samples WHERE recorded_at < $metric_cutoff; SELECT changes(); DELETE FROM idempotency WHERE created_at < $idempotency_cutoff; @@ -254,7 +282,7 @@ public sealed partial class ControlStore(IOptions options) command.Parameters.AddWithValue( "$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O")); command.Parameters.AddWithValue("$now", now.ToString("O")); - var changes = new int[8]; + var changes = new int[9]; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { for (var index = 0; index < changes.Length; index++) @@ -272,8 +300,8 @@ public sealed partial class ControlStore(IOptions options) optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);"; await optimize.ExecuteNonQueryAsync(cancellationToken); return new ControlMaintenanceResult( - changes[2], changes[3], changes[4], changes[5] + changes[6] + changes[7], - changes[0], changes[1]); + changes[3], changes[4], changes[5], changes[6] + changes[7] + changes[8], + changes[0], changes[1] + changes[2]); } public async Task BackupDatabaseAsync(string destinationPath, CancellationToken cancellationToken = default) diff --git a/src/ServerMonitorManager.Control/Program.cs b/src/ServerMonitorManager.Control/Program.cs index ce82667..aae4368 100644 --- a/src/ServerMonitorManager.Control/Program.cs +++ b/src/ServerMonitorManager.Control/Program.cs @@ -531,6 +531,37 @@ control.MapPost("/provisioning/jobs/{id}/retry", async ( return Results.Conflict(new ProblemDetails { Title = exception.Message }); } }); +control.MapPost("/provisioning/jobs/{id}/rollback", async ( + string id, + ProvisioningJobCommandRequest request, + HttpContext context, + ControlStore controlStore, + CancellationToken cancellationToken) => +{ + if (!ProvisioningJobValidator.IsValidId(id) + || !ProvisioningJobValidator.IsValid(request)) + { + return Results.ValidationProblem(new Dictionary + { + ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."] + }); + } + try + { + var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!; + var job = await controlStore.StartProvisioningRollbackAsync( + id, request, actor, cancellationToken); + return job is null ? Results.NotFound() : Results.Ok(job); + } + catch (IdempotencyConflictException) + { + return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" }); + } + catch (ProvisioningTransitionException exception) + { + return Results.Conflict(new ProblemDetails { Title = exception.Message }); + } +}); control.MapPost("/automations/token", async ( AutomationTokenCreateRequest request, HttpContext context, @@ -822,6 +853,9 @@ internal static class ProvisioningJobValidator or ProvisioningJobStates.Completed or ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollingBack + or ProvisioningJobStates.RolledBack + or ProvisioningJobStates.RollbackFailed && request.ProgressPercent is >= 0 and <= 100 && IsSafeCode(request.Step, 64) && IsSafeCode(request.EventCode, 64) diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index c5676bf..1269dc6 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -109,22 +109,29 @@ public sealed partial class ControlStore command.Transaction = transaction; command.CommandText = """ UPDATE provisioning_jobs SET - state = $preflight, + state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END, + current_step = CASE + WHEN state = $queued THEN 'preflight' + ELSE 'rollback' + END, updated_at = $now, version = version + 1 WHERE id = ( SELECT id FROM provisioning_jobs WHERE node_id = $node - AND state = $queued + AND (state = $queued + OR (state = $rolling_back AND current_step = 'rollback-queued')) AND expires_at > $now - ORDER BY created_at, id + ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id LIMIT 1) - AND state = $queued + AND (state = $queued + OR (state = $rolling_back AND current_step = 'rollback-queued')) RETURNING *; """; command.Parameters.AddWithValue("$node", nodeId); command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued); command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight); + command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack); command.Parameters.AddWithValue("$now", now.ToString("O")); ProvisioningJob? job; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) @@ -209,6 +216,7 @@ public sealed partial class ControlStore Version = current.Version + 1, LastError = request.State is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollbackFailed ? request.EventCode : null }; @@ -284,7 +292,8 @@ public sealed partial class ControlStore return null; } if (current.State is not (ProvisioningJobStates.Failed - or ProvisioningJobStates.NeedsReconciliation)) + or ProvisioningJobStates.NeedsReconciliation) + || current.CurrentStep == "rollback-reconcile") { throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued); } @@ -338,6 +347,88 @@ public sealed partial class ControlStore return updated; } + public async Task StartProvisioningRollbackAsync( + string id, + ProvisioningJobCommandRequest request, + string actor, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken); + var operationKey = $"provisioning-rollback:{actor}:{id}:{request.IdempotencyKey}"; + var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest); + var cached = await ReadIdempotentAsync( + connection, transaction, operationKey, requestHash, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + if (cached is not null) + { + await transaction.CommitAsync(cancellationToken); + return cached; + } + + var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken); + if (current is null) + { + await transaction.RollbackAsync(cancellationToken); + return null; + } + if (current.State is not (ProvisioningJobStates.Failed + or ProvisioningJobStates.NeedsReconciliation + or ProvisioningJobStates.RollbackFailed)) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack); + } + + var now = DateTimeOffset.UtcNow; + var lifetime = current.ExpiresAt - current.CreatedAt; + if (lifetime < TimeSpan.FromMinutes(5)) + { + lifetime = TimeSpan.FromMinutes(5); + } + var updated = current with + { + State = ProvisioningJobStates.RollingBack, + ProgressPercent = 0, + CurrentStep = "rollback-queued", + UpdatedAt = now, + ExpiresAt = now.Add(lifetime), + Version = current.Version + 1, + LastError = null + }; + var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = """ + UPDATE provisioning_jobs SET + state = $state, progress_percent = 0, current_step = $step, + updated_at = $updated, expires_at = $expires, + version = $version, last_error = NULL + WHERE id = $id AND version = $previous_version; + """; + update.Parameters.AddWithValue("$state", updated.State); + update.Parameters.AddWithValue("$step", updated.CurrentStep); + update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O")); + update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O")); + update.Parameters.AddWithValue("$version", updated.Version); + update.Parameters.AddWithValue("$id", id); + update.Parameters.AddWithValue("$previous_version", current.Version); + if (await update.ExecuteNonQueryAsync(cancellationToken) != 1) + { + throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack); + } + + await WriteProvisioningEventAsync( + connection, transaction, id, "job.rollback.queued", updated.State, + request.Reason, now, cancellationToken); + await WriteIdempotentAsync( + connection, transaction, operationKey, requestHash, updated, + SmmJsonContext.Default.ProvisioningJob, cancellationToken); + await WriteAuditAsync( + connection, transaction, actor, "provisioning.job.rollback", id, + JsonSerializer.Serialize(new { request.Reason }), cancellationToken); + await transaction.CommitAsync(cancellationToken); + return updated; + } + private async Task TransitionProvisioningJobAsync( string id, ProvisioningJobCommandRequest request, @@ -509,6 +600,10 @@ internal static class ProvisioningStateMachine or ProvisioningJobStates.Running or ProvisioningJobStates.Verifying; } + if (target == ProvisioningJobStates.RollbackFailed) + { + return current == ProvisioningJobStates.RollingBack; + } return (current, target) switch { (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true, @@ -517,6 +612,8 @@ internal static class ProvisioningStateMachine (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100, + (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true, + (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100, _ => false }; } diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index dda378e..a10a631 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -193,5 +193,8 @@ public static class ProvisioningJobStates public const string Completed = "Completed"; public const string Failed = "Failed"; public const string NeedsReconciliation = "NeedsReconciliation"; + public const string RollingBack = "RollingBack"; + public const string RolledBack = "RolledBack"; + public const string RollbackFailed = "RollbackFailed"; public const string Cancelled = "Cancelled"; } diff --git a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs index 7d26549..4954a16 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs @@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable await verify.OpenAsync(cancellationToken); var version = verify.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); + Assert.Equal(4L, (long)(await version.ExecuteScalarAsync(cancellationToken))!); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index 40be431..c21ab3a 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable await migrated.OpenAsync(TestContext.Current.CancellationToken); var version = migrated.CreateCommand(); version.CommandText = "PRAGMA user_version;"; - Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); + Assert.Equal(4L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken))); var table = migrated.CreateCommand(); table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');"; Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken))); @@ -172,6 +172,94 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.Completed, completed!.State); } + [Fact] + public async Task FailedJobRollbackIsNodeScopedRecoverableAndTerminal() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = CreateStore(); + await store.InitializeAsync(cancellationToken); + await EnrollAgentAsync(store, "home", "7788", cancellationToken); + using var parameters = JsonDocument.Parse("{}"); + var created = await store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Rollback test", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + var failed = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.Failed, 20, "inspect-os", "preflight.failed", + "Preflight failed.", Guid.NewGuid().ToString()), + cancellationToken); + Assert.Equal(ProvisioningJobStates.Failed, failed!.State); + await Assert.ThrowsAsync(() => store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Must remain blocked", Guid.NewGuid().ToString()), + "operator", + cancellationToken)); + + var rollbackRequest = new ProvisioningJobCommandRequest( + "Restore preflight state", Guid.NewGuid().ToString()); + var queued = await store.StartProvisioningRollbackAsync( + created.Id, rollbackRequest, "operator", cancellationToken); + var replay = await store.StartProvisioningRollbackAsync( + created.Id, rollbackRequest, "operator", cancellationToken); + Assert.Equal(queued!.Id, replay!.Id); + Assert.Equal("rollback-queued", queued.CurrentStep); + var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken); + Assert.Equal(ProvisioningJobStates.RollingBack, claimed!.State); + Assert.Equal("rollback", claimed.CurrentStep); + Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + + var maintenance = await store.MaintainAsync( + queued.ExpiresAt.AddSeconds(1), cancellationToken); + Assert.Equal(1, maintenance.ProvisioningJobsNeedingReconciliation); + var uncertain = await store.GetProvisioningJobAsync(created.Id, cancellationToken); + Assert.Equal(ProvisioningJobStates.NeedsReconciliation, uncertain!.State); + Assert.Equal("rollback-reconcile", uncertain.CurrentStep); + await Assert.ThrowsAsync(() => + store.RetryProvisioningJobAsync( + created.Id, + new ProvisioningJobCommandRequest("Unsafe retry", Guid.NewGuid().ToString()), + "operator", + cancellationToken)); + + await store.StartProvisioningRollbackAsync( + created.Id, + new ProvisioningJobCommandRequest("Resume rollback", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); + var rolling = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RollingBack, 50, "restore", "rollback.progress", + "Restoring backup.", Guid.NewGuid().ToString()), + cancellationToken); + var rolledBack = await store.ReportProvisioningProgressAsync( + "home", created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RolledBack, 100, "rollback-complete", "rollback.completed", + "Backup restored.", Guid.NewGuid().ToString()), + cancellationToken); + Assert.Equal(50, rolling!.ProgressPercent); + Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State); + Assert.Null(rolledBack.LastError); + var replacement = await store.CreateProvisioningJobAsync( + "home", + new ProvisioningJobCreateRequest( + "preflight", 1, parameters.RootElement.Clone(), 5, + "Allowed after rollback", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.Equal(ProvisioningJobStates.Queued, replacement.State); + } + [Fact] public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates() {