Add provisioning rollback workflow

2026-07-19 22:10:42 +07:00 · 2026-07-19 22:10:42 +07:00 · 3ae9f4717b
commit 3ae9f4717b
parent 41d33a5ce0
7 changed files with 262 additions and 12 deletions
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@ -109,7 +109,7 @@
 ## Этап 8 — Provisioning control plane

 - [x] модели и SQLite migration v2 для ProvisioningJob;
- [ ] state machine, confirmations, cancellation, retry и rollback;
+- [x] state machine, confirmations, cancellation, retry и rollback;
 - [x] создание, чтение, подтверждение и отмена через Operator API;
 - [ ] выполнение, retry, verification и rollback в полной state machine;
 - [x] обязательные idempotency key, audit reason и job TTL;
--- a/src/ServerMonitorManager.Control/ControlStore.cs
+++ b/src/ServerMonitorManager.Control/ControlStore.cs
@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control;

 public sealed partial class ControlStore(IOptions<ControlOptions> options)
 {
-    private const int CurrentSchemaVersion = 3;
+    private const int CurrentSchemaVersion = 4;
    private readonly ControlOptions _options = options.Value;
    private readonly string _connectionString = new SqliteConnectionStringBuilder
    {
@ -196,6 +196,23 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
            await migrateProgress.ExecuteNonQueryAsync(cancellationToken);
            await migration.CommitAsync(cancellationToken);
        }
+
+        if (schemaVersion < 4)
+        {
+            await using var migration =
+                (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+            var strengthenProvisioningLock = connection.CreateCommand();
+            strengthenProvisioningLock.Transaction = migration;
+            strengthenProvisioningLock.CommandText = """
+                DROP INDEX IF EXISTS ux_provisioning_jobs_active_node;
+                CREATE UNIQUE INDEX ux_provisioning_jobs_active_node
+                    ON provisioning_jobs(node_id)
+                    WHERE state NOT IN ('Completed', 'Cancelled', 'RolledBack');
+                PRAGMA user_version = 4;
+                """;
+            await strengthenProvisioningLock.ExecuteNonQueryAsync(cancellationToken);
+            await migration.CommitAsync(cancellationToken);
+        }
    }

    public async Task<ControlMaintenanceResult> MaintainAsync(
@ -234,6 +251,17 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
            WHERE expires_at <= $now
              AND state IN ('Preflight', 'Running', 'Verifying');
            SELECT changes();
+            INSERT INTO provisioning_events(job_id, recorded_at, event_type, state, message)
+            SELECT id, $now, 'job.rollback.reconciliation.required', 'NeedsReconciliation',
+                   'Rollback lease expired; factual state must be inspected.'
+            FROM provisioning_jobs
+            WHERE expires_at <= $now AND state = 'RollingBack';
+            UPDATE provisioning_jobs SET
+                state = 'NeedsReconciliation', updated_at = $now,
+                current_step = 'rollback-reconcile', version = version + 1,
+                last_error = 'job.rollback.ttl_expired'
+            WHERE expires_at <= $now AND state = 'RollingBack';
+            SELECT changes();
            DELETE FROM metric_samples WHERE recorded_at < $metric_cutoff;
            SELECT changes();
            DELETE FROM idempotency WHERE created_at < $idempotency_cutoff;
@ -254,7 +282,7 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
        command.Parameters.AddWithValue(
            "$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O"));
        command.Parameters.AddWithValue("$now", now.ToString("O"));
-        var changes = new int[8];
+        var changes = new int[9];
        await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
        {
            for (var index = 0; index < changes.Length; index++)
@ -272,8 +300,8 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
        optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);";
        await optimize.ExecuteNonQueryAsync(cancellationToken);
        return new ControlMaintenanceResult(
-            changes[2], changes[3], changes[4], changes[5] + changes[6] + changes[7],
-            changes[0], changes[1]);
+            changes[3], changes[4], changes[5], changes[6] + changes[7] + changes[8],
+            changes[0], changes[1] + changes[2]);
    }

    public async Task BackupDatabaseAsync(string destinationPath, CancellationToken cancellationToken = default)
--- a/src/ServerMonitorManager.Control/Program.cs
+++ b/src/ServerMonitorManager.Control/Program.cs
@ -531,6 +531,37 @@ control.MapPost("/provisioning/jobs/{id}/retry", async (
        return Results.Conflict(new ProblemDetails { Title = exception.Message });
    }
 });
+control.MapPost("/provisioning/jobs/{id}/rollback", async (
+    string id,
+    ProvisioningJobCommandRequest request,
+    HttpContext context,
+    ControlStore controlStore,
+    CancellationToken cancellationToken) =>
+{
+    if (!ProvisioningJobValidator.IsValidId(id)
+        || !ProvisioningJobValidator.IsValid(request))
+    {
+        return Results.ValidationProblem(new Dictionary<string, string[]>
+        {
+            ["provisioningJob"] = ["Invalid job id, reason, or idempotency key."]
+        });
+    }
+    try
+    {
+        var actor = context.User.FindFirstValue(ClaimTypes.NameIdentifier)!;
+        var job = await controlStore.StartProvisioningRollbackAsync(
+            id, request, actor, cancellationToken);
+        return job is null ? Results.NotFound() : Results.Ok(job);
+    }
+    catch (IdempotencyConflictException)
+    {
+        return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
+    }
+    catch (ProvisioningTransitionException exception)
+    {
+        return Results.Conflict(new ProblemDetails { Title = exception.Message });
+    }
+});
 control.MapPost("/automations/token", async (
    AutomationTokenCreateRequest request,
    HttpContext context,
@ -822,6 +853,9 @@ internal static class ProvisioningJobValidator
            or ProvisioningJobStates.Completed
            or ProvisioningJobStates.Failed
            or ProvisioningJobStates.NeedsReconciliation
+            or ProvisioningJobStates.RollingBack
+            or ProvisioningJobStates.RolledBack
+            or ProvisioningJobStates.RollbackFailed
           && request.ProgressPercent is >= 0 and <= 100
           && IsSafeCode(request.Step, 64)
           && IsSafeCode(request.EventCode, 64)
--- a/src/ServerMonitorManager.Control/ProvisioningStore.cs
+++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs
@ -109,22 +109,29 @@ public sealed partial class ControlStore
        command.Transaction = transaction;
        command.CommandText = """
            UPDATE provisioning_jobs SET
-                state = $preflight,
+                state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END,
+                current_step = CASE
+                    WHEN state = $queued THEN 'preflight'
+                    ELSE 'rollback'
+                END,
                updated_at = $now,
                version = version + 1
            WHERE id = (
                SELECT id FROM provisioning_jobs
                WHERE node_id = $node
-                  AND state = $queued
+                  AND (state = $queued
+                       OR (state = $rolling_back AND current_step = 'rollback-queued'))
                  AND expires_at > $now
-                ORDER BY created_at, id
+                ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id
                LIMIT 1)
-              AND state = $queued
+              AND (state = $queued
+                   OR (state = $rolling_back AND current_step = 'rollback-queued'))
            RETURNING *;
            """;
        command.Parameters.AddWithValue("$node", nodeId);
        command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued);
        command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight);
+        command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack);
        command.Parameters.AddWithValue("$now", now.ToString("O"));
        ProvisioningJob? job;
        await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
@ -209,6 +216,7 @@ public sealed partial class ControlStore
            Version = current.Version + 1,
            LastError = request.State is ProvisioningJobStates.Failed
                or ProvisioningJobStates.NeedsReconciliation
+                or ProvisioningJobStates.RollbackFailed
                ? request.EventCode
                : null
        };
@ -284,7 +292,8 @@ public sealed partial class ControlStore
            return null;
        }
        if (current.State is not (ProvisioningJobStates.Failed
-            or ProvisioningJobStates.NeedsReconciliation))
+            or ProvisioningJobStates.NeedsReconciliation)
+            || current.CurrentStep == "rollback-reconcile")
        {
            throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.Queued);
        }
@ -338,6 +347,88 @@ public sealed partial class ControlStore
        return updated;
    }

+    public async Task<ProvisioningJob?> StartProvisioningRollbackAsync(
+        string id,
+        ProvisioningJobCommandRequest request,
+        string actor,
+        CancellationToken cancellationToken = default)
+    {
+        await using var connection = await OpenAsync(cancellationToken);
+        await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
+        var operationKey = $"provisioning-rollback:{actor}:{id}:{request.IdempotencyKey}";
+        var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobCommandRequest);
+        var cached = await ReadIdempotentAsync(
+            connection, transaction, operationKey, requestHash,
+            SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+        if (cached is not null)
+        {
+            await transaction.CommitAsync(cancellationToken);
+            return cached;
+        }
+
+        var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
+        if (current is null)
+        {
+            await transaction.RollbackAsync(cancellationToken);
+            return null;
+        }
+        if (current.State is not (ProvisioningJobStates.Failed
+            or ProvisioningJobStates.NeedsReconciliation
+            or ProvisioningJobStates.RollbackFailed))
+        {
+            throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack);
+        }
+
+        var now = DateTimeOffset.UtcNow;
+        var lifetime = current.ExpiresAt - current.CreatedAt;
+        if (lifetime < TimeSpan.FromMinutes(5))
+        {
+            lifetime = TimeSpan.FromMinutes(5);
+        }
+        var updated = current with
+        {
+            State = ProvisioningJobStates.RollingBack,
+            ProgressPercent = 0,
+            CurrentStep = "rollback-queued",
+            UpdatedAt = now,
+            ExpiresAt = now.Add(lifetime),
+            Version = current.Version + 1,
+            LastError = null
+        };
+        var update = connection.CreateCommand();
+        update.Transaction = transaction;
+        update.CommandText = """
+            UPDATE provisioning_jobs SET
+                state = $state, progress_percent = 0, current_step = $step,
+                updated_at = $updated, expires_at = $expires,
+                version = $version, last_error = NULL
+            WHERE id = $id AND version = $previous_version;
+            """;
+        update.Parameters.AddWithValue("$state", updated.State);
+        update.Parameters.AddWithValue("$step", updated.CurrentStep);
+        update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O"));
+        update.Parameters.AddWithValue("$expires", updated.ExpiresAt.ToString("O"));
+        update.Parameters.AddWithValue("$version", updated.Version);
+        update.Parameters.AddWithValue("$id", id);
+        update.Parameters.AddWithValue("$previous_version", current.Version);
+        if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
+        {
+            throw new ProvisioningTransitionException(current.State, ProvisioningJobStates.RollingBack);
+        }
+
+        await WriteProvisioningEventAsync(
+            connection, transaction, id, "job.rollback.queued", updated.State,
+            request.Reason, now, cancellationToken);
+        await WriteIdempotentAsync(
+            connection, transaction, operationKey, requestHash, updated,
+            SmmJsonContext.Default.ProvisioningJob, cancellationToken);
+        await WriteAuditAsync(
+            connection, transaction, actor, "provisioning.job.rollback", id,
+            JsonSerializer.Serialize(new { request.Reason }), cancellationToken);
+        await transaction.CommitAsync(cancellationToken);
+        return updated;
+    }
+
    private async Task<ProvisioningJob?> TransitionProvisioningJobAsync(
        string id,
        ProvisioningJobCommandRequest request,
@ -509,6 +600,10 @@ internal static class ProvisioningStateMachine
                or ProvisioningJobStates.Running
                or ProvisioningJobStates.Verifying;
        }
+        if (target == ProvisioningJobStates.RollbackFailed)
+        {
+            return current == ProvisioningJobStates.RollingBack;
+        }
        return (current, target) switch
        {
            (ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true,
@ -517,6 +612,8 @@ internal static class ProvisioningStateMachine
            (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true,
            (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true,
            (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100,
+            (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true,
+            (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100,
            _ => false
        };
    }
--- a/src/ServerMonitorManager.Core/Contracts.cs
+++ b/src/ServerMonitorManager.Core/Contracts.cs
@ -193,5 +193,8 @@ public static class ProvisioningJobStates
    public const string Completed = "Completed";
    public const string Failed = "Failed";
    public const string NeedsReconciliation = "NeedsReconciliation";
+    public const string RollingBack = "RollingBack";
+    public const string RolledBack = "RolledBack";
+    public const string RollbackFailed = "RollbackFailed";
    public const string Cancelled = "Cancelled";
 }
--- a/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/ControlMaintenanceTests.cs
@ -106,7 +106,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
        await verify.OpenAsync(cancellationToken);
        var version = verify.CreateCommand();
        version.CommandText = "PRAGMA user_version;";
-        Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
+        Assert.Equal(4L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
    }

    [Fact]
--- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs
+++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs
@ -31,7 +31,7 @@ public sealed class ControlStoreTests : IAsyncDisposable
        await migrated.OpenAsync(TestContext.Current.CancellationToken);
        var version = migrated.CreateCommand();
        version.CommandText = "PRAGMA user_version;";
-        Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
+        Assert.Equal(4L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
        var table = migrated.CreateCommand();
        table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');";
        Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
@ -172,6 +172,94 @@ public sealed class ControlStoreTests : IAsyncDisposable
        Assert.Equal(ProvisioningJobStates.Completed, completed!.State);
    }

+    [Fact]
+    public async Task FailedJobRollbackIsNodeScopedRecoverableAndTerminal()
+    {
+        var cancellationToken = TestContext.Current.CancellationToken;
+        var store = CreateStore();
+        await store.InitializeAsync(cancellationToken);
+        await EnrollAgentAsync(store, "home", "7788", cancellationToken);
+        using var parameters = JsonDocument.Parse("{}");
+        var created = await store.CreateProvisioningJobAsync(
+            "home",
+            new ProvisioningJobCreateRequest(
+                "preflight", 1, parameters.RootElement.Clone(), 5,
+                "Rollback test", Guid.NewGuid().ToString()),
+            "operator",
+            cancellationToken);
+        Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+        var failed = await store.ReportProvisioningProgressAsync(
+            "home", created.Id,
+            new ProvisioningJobProgressRequest(
+                ProvisioningJobStates.Failed, 20, "inspect-os", "preflight.failed",
+                "Preflight failed.", Guid.NewGuid().ToString()),
+            cancellationToken);
+        Assert.Equal(ProvisioningJobStates.Failed, failed!.State);
+        await Assert.ThrowsAsync<SqliteException>(() => store.CreateProvisioningJobAsync(
+            "home",
+            new ProvisioningJobCreateRequest(
+                "preflight", 1, parameters.RootElement.Clone(), 5,
+                "Must remain blocked", Guid.NewGuid().ToString()),
+            "operator",
+            cancellationToken));
+
+        var rollbackRequest = new ProvisioningJobCommandRequest(
+            "Restore preflight state", Guid.NewGuid().ToString());
+        var queued = await store.StartProvisioningRollbackAsync(
+            created.Id, rollbackRequest, "operator", cancellationToken);
+        var replay = await store.StartProvisioningRollbackAsync(
+            created.Id, rollbackRequest, "operator", cancellationToken);
+        Assert.Equal(queued!.Id, replay!.Id);
+        Assert.Equal("rollback-queued", queued.CurrentStep);
+        var claimed = await store.ClaimNextProvisioningJobAsync("home", cancellationToken);
+        Assert.Equal(ProvisioningJobStates.RollingBack, claimed!.State);
+        Assert.Equal("rollback", claimed.CurrentStep);
+        Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+
+        var maintenance = await store.MaintainAsync(
+            queued.ExpiresAt.AddSeconds(1), cancellationToken);
+        Assert.Equal(1, maintenance.ProvisioningJobsNeedingReconciliation);
+        var uncertain = await store.GetProvisioningJobAsync(created.Id, cancellationToken);
+        Assert.Equal(ProvisioningJobStates.NeedsReconciliation, uncertain!.State);
+        Assert.Equal("rollback-reconcile", uncertain.CurrentStep);
+        await Assert.ThrowsAsync<ProvisioningTransitionException>(() =>
+            store.RetryProvisioningJobAsync(
+                created.Id,
+                new ProvisioningJobCommandRequest("Unsafe retry", Guid.NewGuid().ToString()),
+                "operator",
+                cancellationToken));
+
+        await store.StartProvisioningRollbackAsync(
+            created.Id,
+            new ProvisioningJobCommandRequest("Resume rollback", Guid.NewGuid().ToString()),
+            "operator",
+            cancellationToken);
+        Assert.NotNull(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
+        var rolling = await store.ReportProvisioningProgressAsync(
+            "home", created.Id,
+            new ProvisioningJobProgressRequest(
+                ProvisioningJobStates.RollingBack, 50, "restore", "rollback.progress",
+                "Restoring backup.", Guid.NewGuid().ToString()),
+            cancellationToken);
+        var rolledBack = await store.ReportProvisioningProgressAsync(
+            "home", created.Id,
+            new ProvisioningJobProgressRequest(
+                ProvisioningJobStates.RolledBack, 100, "rollback-complete", "rollback.completed",
+                "Backup restored.", Guid.NewGuid().ToString()),
+            cancellationToken);
+        Assert.Equal(50, rolling!.ProgressPercent);
+        Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State);
+        Assert.Null(rolledBack.LastError);
+        var replacement = await store.CreateProvisioningJobAsync(
+            "home",
+            new ProvisioningJobCreateRequest(
+                "preflight", 1, parameters.RootElement.Clone(), 5,
+                "Allowed after rollback", Guid.NewGuid().ToString()),
+            "operator",
+            cancellationToken);
+        Assert.Equal(ProvisioningJobStates.Queued, replacement.State);
+    }
+
    [Fact]
    public void DiagnosticsExportOmitsRawIdentitiesAndNormalizesStates()
    {