Execute confirmed timezone provisioning safely #6
8 changed files with 295 additions and 5 deletions
|
|
@ -10,7 +10,7 @@ namespace ServerMonitorManager.Control;
|
|||
|
||||
public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||
{
|
||||
private const int CurrentSchemaVersion = 2;
|
||||
private const int CurrentSchemaVersion = 3;
|
||||
private readonly ControlOptions _options = options.Value;
|
||||
private readonly string _connectionString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
|
|
@ -179,6 +179,23 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
|||
await migrateProvisioning.ExecuteNonQueryAsync(cancellationToken);
|
||||
await migration.CommitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
if (schemaVersion < 3)
|
||||
{
|
||||
await using var migration =
|
||||
(SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||
var migrateProgress = connection.CreateCommand();
|
||||
migrateProgress.Transaction = migration;
|
||||
migrateProgress.CommandText = """
|
||||
ALTER TABLE provisioning_jobs
|
||||
ADD COLUMN progress_percent INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE provisioning_jobs
|
||||
ADD COLUMN current_step TEXT NOT NULL DEFAULT '';
|
||||
PRAGMA user_version = 3;
|
||||
""";
|
||||
await migrateProgress.ExecuteNonQueryAsync(cancellationToken);
|
||||
await migration.CommitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<ControlMaintenanceResult> MaintainAsync(
|
||||
|
|
|
|||
|
|
@ -391,6 +391,41 @@ agents.MapGet("/provisioning/jobs/next", async (
|
|||
var job = await controlStore.ClaimNextProvisioningJobAsync(nodeId, cancellationToken);
|
||||
return job is null ? Results.NoContent() : Results.Ok(job);
|
||||
});
|
||||
agents.MapPost("/provisioning/jobs/{id}/progress", async (
|
||||
string id,
|
||||
ProvisioningJobProgressRequest request,
|
||||
HttpContext context,
|
||||
ControlStore controlStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var nodeId = context.User.FindFirstValue(ClaimTypes.NameIdentifier);
|
||||
if (string.IsNullOrWhiteSpace(nodeId)
|
||||
|| !NodeIdValidator.IsValid(nodeId)
|
||||
|| !ProvisioningJobValidator.IsValidId(id)
|
||||
|| !ProvisioningJobValidator.IsValid(request))
|
||||
{
|
||||
return Results.ValidationProblem(new Dictionary<string, string[]>
|
||||
{
|
||||
["provisioningProgress"] =
|
||||
["Invalid job id, state, progress, step, event code, message, or idempotency key."]
|
||||
});
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var job = await controlStore.ReportProvisioningProgressAsync(
|
||||
nodeId, id, request, cancellationToken);
|
||||
return job is null ? Results.NotFound() : Results.Ok(job);
|
||||
}
|
||||
catch (IdempotencyConflictException)
|
||||
{
|
||||
return Results.Conflict(new ProblemDetails { Title = "Idempotency key conflict" });
|
||||
}
|
||||
catch (ProvisioningTransitionException exception)
|
||||
{
|
||||
return Results.Conflict(new ProblemDetails { Title = exception.Message });
|
||||
}
|
||||
});
|
||||
|
||||
var control = app.MapGroup("/api/v1/control").RequireAuthorization("Operator");
|
||||
control.MapGet("/agents", async (ControlStore controlStore, CancellationToken cancellationToken) =>
|
||||
|
|
@ -749,9 +784,28 @@ internal static class ProvisioningJobValidator
|
|||
=> request.Reason.Length is >= 1 and <= 256
|
||||
&& IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
|
||||
|
||||
public static bool IsValid(ProvisioningJobProgressRequest request)
|
||||
=> request.State is ProvisioningJobStates.Preflight
|
||||
or ProvisioningJobStates.Running
|
||||
or ProvisioningJobStates.Verifying
|
||||
or ProvisioningJobStates.Completed
|
||||
or ProvisioningJobStates.Failed
|
||||
or ProvisioningJobStates.NeedsReconciliation
|
||||
&& request.ProgressPercent is >= 0 and <= 100
|
||||
&& IsSafeCode(request.Step, 64)
|
||||
&& IsSafeCode(request.EventCode, 64)
|
||||
&& request.Message.Length <= 512
|
||||
&& IdempotencyKeyValidator.IsValid(request.IdempotencyKey);
|
||||
|
||||
public static bool IsValidId(string id)
|
||||
=> id.Length == 32 && Guid.TryParseExact(id, "N", out _);
|
||||
|
||||
public static bool RequiresConfirmation(string actionType)
|
||||
=> actionType == "system.base-install";
|
||||
|
||||
private static bool IsSafeCode(string value, int maximumLength)
|
||||
=> value.Length is >= 1 && value.Length <= maximumLength
|
||||
&& value.All(character => character is >= 'a' and <= 'z'
|
||||
or >= '0' and <= '9'
|
||||
or '.' or '-' or '_');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ public sealed partial class ControlStore
|
|||
null,
|
||||
null,
|
||||
1,
|
||||
0,
|
||||
"queued",
|
||||
null);
|
||||
|
||||
var insert = connection.CreateCommand();
|
||||
|
|
@ -166,6 +168,96 @@ public sealed partial class ControlStore
|
|||
"job.cancelled", "provisioning.job.cancelled",
|
||||
setConfirmedAt: false, cancellationToken);
|
||||
|
||||
public async Task<ProvisioningJob?> ReportProvisioningProgressAsync(
|
||||
string nodeId,
|
||||
string id,
|
||||
ProvisioningJobProgressRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||
var operationKey = $"provisioning-progress:{nodeId}:{id}:{request.IdempotencyKey}";
|
||||
var requestHash = Fingerprint(request, SmmJsonContext.Default.ProvisioningJobProgressRequest);
|
||||
var cached = await ReadIdempotentAsync(
|
||||
connection, transaction, operationKey, requestHash,
|
||||
SmmJsonContext.Default.ProvisioningJob, cancellationToken);
|
||||
if (cached is not null)
|
||||
{
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return cached;
|
||||
}
|
||||
|
||||
var current = await ReadProvisioningJobAsync(connection, transaction, id, cancellationToken);
|
||||
if (current is null || !string.Equals(current.NodeId, nodeId, StringComparison.Ordinal))
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
return null;
|
||||
}
|
||||
if (!ProvisioningStateMachine.CanReport(
|
||||
current.State, request.State, current.ProgressPercent, request.ProgressPercent))
|
||||
{
|
||||
throw new ProvisioningTransitionException(current.State, request.State);
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var updated = current with
|
||||
{
|
||||
State = request.State,
|
||||
ProgressPercent = request.ProgressPercent,
|
||||
CurrentStep = request.Step,
|
||||
UpdatedAt = now,
|
||||
Version = current.Version + 1,
|
||||
LastError = request.State is ProvisioningJobStates.Failed
|
||||
or ProvisioningJobStates.NeedsReconciliation
|
||||
? request.EventCode
|
||||
: null
|
||||
};
|
||||
var update = connection.CreateCommand();
|
||||
update.Transaction = transaction;
|
||||
update.CommandText = """
|
||||
UPDATE provisioning_jobs SET
|
||||
state = $state,
|
||||
progress_percent = $progress,
|
||||
current_step = $step,
|
||||
updated_at = $updated,
|
||||
version = $version,
|
||||
last_error = $error
|
||||
WHERE id = $id AND node_id = $node AND version = $previous_version;
|
||||
""";
|
||||
update.Parameters.AddWithValue("$state", updated.State);
|
||||
update.Parameters.AddWithValue("$progress", updated.ProgressPercent);
|
||||
update.Parameters.AddWithValue("$step", updated.CurrentStep);
|
||||
update.Parameters.AddWithValue("$updated", updated.UpdatedAt.ToString("O"));
|
||||
update.Parameters.AddWithValue("$version", updated.Version);
|
||||
update.Parameters.AddWithValue("$error", (object?)updated.LastError ?? DBNull.Value);
|
||||
update.Parameters.AddWithValue("$id", updated.Id);
|
||||
update.Parameters.AddWithValue("$node", nodeId);
|
||||
update.Parameters.AddWithValue("$previous_version", current.Version);
|
||||
if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
|
||||
{
|
||||
throw new ProvisioningTransitionException(current.State, request.State);
|
||||
}
|
||||
|
||||
await WriteProvisioningEventAsync(
|
||||
connection, transaction, id, request.EventCode, request.State,
|
||||
request.Message, now, cancellationToken);
|
||||
await WriteIdempotentAsync(
|
||||
connection, transaction, operationKey, requestHash, updated,
|
||||
SmmJsonContext.Default.ProvisioningJob, cancellationToken);
|
||||
await WriteAuditAsync(
|
||||
connection, transaction, nodeId, "provisioning.job.progress", id,
|
||||
JsonSerializer.Serialize(new
|
||||
{
|
||||
request.State,
|
||||
request.ProgressPercent,
|
||||
request.Step,
|
||||
request.EventCode
|
||||
}),
|
||||
cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<ProvisioningJob?> TransitionProvisioningJobAsync(
|
||||
string id,
|
||||
ProvisioningJobCommandRequest request,
|
||||
|
|
@ -265,7 +357,8 @@ public sealed partial class ControlStore
|
|||
DateTimeOffset.Parse(reader.GetString(11)),
|
||||
reader.IsDBNull(12) ? null : DateTimeOffset.Parse(reader.GetString(12)),
|
||||
reader.IsDBNull(13) ? null : DateTimeOffset.Parse(reader.GetString(13)),
|
||||
reader.GetInt64(14), reader.IsDBNull(15) ? null : reader.GetString(15));
|
||||
reader.GetInt64(14), reader.GetInt32(16), reader.GetString(17),
|
||||
reader.IsDBNull(15) ? null : reader.GetString(15));
|
||||
|
||||
private static JsonElement ParseParameters(string json)
|
||||
{
|
||||
|
|
@ -321,3 +414,30 @@ public sealed class ProvisioningNodeNotFoundException(string nodeId) : Exception
|
|||
|
||||
public sealed class ProvisioningTransitionException(string state, string targetState)
|
||||
: Exception($"Provisioning job cannot transition from '{state}' to '{targetState}'.");
|
||||
|
||||
internal static class ProvisioningStateMachine
|
||||
{
|
||||
public static bool CanReport(string current, string target, int currentProgress, int targetProgress)
|
||||
{
|
||||
if (targetProgress < currentProgress || targetProgress is < 0 or > 100)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (target is ProvisioningJobStates.Failed or ProvisioningJobStates.NeedsReconciliation)
|
||||
{
|
||||
return current is ProvisioningJobStates.Preflight
|
||||
or ProvisioningJobStates.Running
|
||||
or ProvisioningJobStates.Verifying;
|
||||
}
|
||||
return (current, target) switch
|
||||
{
|
||||
(ProvisioningJobStates.Preflight, ProvisioningJobStates.Preflight) => true,
|
||||
(ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true,
|
||||
(ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true,
|
||||
(ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true,
|
||||
(ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true,
|
||||
(ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,6 +155,14 @@ public sealed record ProvisioningJobCommandRequest(
|
|||
string Reason,
|
||||
string IdempotencyKey);
|
||||
|
||||
public sealed record ProvisioningJobProgressRequest(
|
||||
string State,
|
||||
int ProgressPercent,
|
||||
string Step,
|
||||
string EventCode,
|
||||
string Message,
|
||||
string IdempotencyKey);
|
||||
|
||||
public sealed record ProvisioningJob(
|
||||
string Id,
|
||||
string NodeId,
|
||||
|
|
@ -171,6 +179,8 @@ public sealed record ProvisioningJob(
|
|||
DateTimeOffset? ConfirmedAt,
|
||||
DateTimeOffset? CancelledAt,
|
||||
long Version,
|
||||
int ProgressPercent,
|
||||
string CurrentStep,
|
||||
string? LastError);
|
||||
|
||||
public static class ProvisioningJobStates
|
||||
|
|
@ -178,5 +188,10 @@ public static class ProvisioningJobStates
|
|||
public const string Queued = "Queued";
|
||||
public const string Preflight = "Preflight";
|
||||
public const string AwaitingConfirmation = "AwaitingConfirmation";
|
||||
public const string Running = "Running";
|
||||
public const string Verifying = "Verifying";
|
||||
public const string Completed = "Completed";
|
||||
public const string Failed = "Failed";
|
||||
public const string NeedsReconciliation = "NeedsReconciliation";
|
||||
public const string Cancelled = "Cancelled";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ namespace ServerMonitorManager.Core;
|
|||
[JsonSerializable(typeof(ControlError))]
|
||||
[JsonSerializable(typeof(ProvisioningJobCreateRequest))]
|
||||
[JsonSerializable(typeof(ProvisioningJobCommandRequest))]
|
||||
[JsonSerializable(typeof(ProvisioningJobProgressRequest))]
|
||||
[JsonSerializable(typeof(ProvisioningJob))]
|
||||
[JsonSerializable(typeof(ProvisioningJob[]))]
|
||||
public sealed partial class SmmJsonContext : JsonSerializerContext;
|
||||
|
|
|
|||
|
|
@ -160,6 +160,29 @@ public sealed class ControlApiTests : IAsyncDisposable
|
|||
Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Preflight, claimed.State);
|
||||
Assert.Equal(HttpStatusCode.NoContent, (await assigned.GetAsync(
|
||||
"/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);
|
||||
|
||||
var progress = new
|
||||
{
|
||||
state = "Running",
|
||||
progressPercent = 25,
|
||||
step = "apply",
|
||||
eventCode = "apply.started",
|
||||
message = "Applying the approved plan.",
|
||||
idempotencyKey = Guid.NewGuid().ToString()
|
||||
};
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await other.PostAsJsonAsync(
|
||||
$"/api/v1/agents/provisioning/jobs/{created.Id}/progress",
|
||||
progress,
|
||||
cancellationToken)).StatusCode);
|
||||
using var progressResponse = await assigned.PostAsJsonAsync(
|
||||
$"/api/v1/agents/provisioning/jobs/{created.Id}/progress",
|
||||
progress,
|
||||
cancellationToken);
|
||||
Assert.Equal(HttpStatusCode.OK, progressResponse.StatusCode);
|
||||
var updated = await progressResponse.Content.ReadFromJsonAsync<ServerMonitorManager.Core.ProvisioningJob>(
|
||||
cancellationToken);
|
||||
Assert.Equal(25, updated!.ProgressPercent);
|
||||
Assert.Equal("apply", updated.CurrentStep);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync() => await _factory.DisposeAsync();
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
|
|||
await verify.OpenAsync(cancellationToken);
|
||||
var version = verify.CreateCommand();
|
||||
version.CommandText = "PRAGMA user_version;";
|
||||
Assert.Equal(2L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
|
||||
Assert.Equal(3L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
|
|||
|
|
@ -31,10 +31,10 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
|||
await migrated.OpenAsync(TestContext.Current.CancellationToken);
|
||||
var version = migrated.CreateCommand();
|
||||
version.CommandText = "PRAGMA user_version;";
|
||||
Assert.Equal(2L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
|
||||
Assert.Equal(3L, Convert.ToInt64(await version.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
|
||||
var table = migrated.CreateCommand();
|
||||
table.CommandText = "SELECT COUNT(*) FROM pragma_table_info('provisioning_jobs');";
|
||||
Assert.Equal(16L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
|
||||
Assert.Equal(18L, Convert.ToInt64(await table.ExecuteScalarAsync(TestContext.Current.CancellationToken)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
@ -110,6 +110,66 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
|||
Assert.Equal("home", claimed.NodeId);
|
||||
Assert.Equal(ProvisioningJobStates.Preflight, claimed.State);
|
||||
Assert.Null(Assert.Single(claims, job => job is null));
|
||||
|
||||
var preflight = new ProvisioningJobProgressRequest(
|
||||
ProvisioningJobStates.Preflight, 10, "inspect-os", "preflight.progress",
|
||||
"Operating system detected.", Guid.NewGuid().ToString());
|
||||
Assert.Null(await store.ReportProvisioningProgressAsync(
|
||||
"other", created.Id, preflight, cancellationToken));
|
||||
var firstProgress = await store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id, preflight, cancellationToken);
|
||||
var replay = await store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id, preflight, cancellationToken);
|
||||
Assert.Equal(firstProgress!.Id, replay!.Id);
|
||||
Assert.Equal(10, replay.ProgressPercent);
|
||||
Assert.Equal("inspect-os", replay.CurrentStep);
|
||||
await Assert.ThrowsAsync<ProvisioningTransitionException>(() =>
|
||||
store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id,
|
||||
preflight with
|
||||
{
|
||||
State = ProvisioningJobStates.Completed,
|
||||
ProgressPercent = 100,
|
||||
IdempotencyKey = Guid.NewGuid().ToString()
|
||||
},
|
||||
cancellationToken));
|
||||
|
||||
var running = await store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id,
|
||||
preflight with
|
||||
{
|
||||
State = ProvisioningJobStates.Running,
|
||||
ProgressPercent = 40,
|
||||
Step = "apply",
|
||||
EventCode = "apply.started",
|
||||
IdempotencyKey = Guid.NewGuid().ToString()
|
||||
},
|
||||
cancellationToken);
|
||||
var verifying = await store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id,
|
||||
preflight with
|
||||
{
|
||||
State = ProvisioningJobStates.Verifying,
|
||||
ProgressPercent = 90,
|
||||
Step = "verify",
|
||||
EventCode = "verify.started",
|
||||
IdempotencyKey = Guid.NewGuid().ToString()
|
||||
},
|
||||
cancellationToken);
|
||||
var completed = await store.ReportProvisioningProgressAsync(
|
||||
"home", created.Id,
|
||||
preflight with
|
||||
{
|
||||
State = ProvisioningJobStates.Completed,
|
||||
ProgressPercent = 100,
|
||||
Step = "complete",
|
||||
EventCode = "job.completed",
|
||||
IdempotencyKey = Guid.NewGuid().ToString()
|
||||
},
|
||||
cancellationToken);
|
||||
Assert.Equal(ProvisioningJobStates.Running, running!.State);
|
||||
Assert.Equal(ProvisioningJobStates.Verifying, verifying!.State);
|
||||
Assert.Equal(ProvisioningJobStates.Completed, completed!.State);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
|
|
|||
Loading…
Reference in a new issue