From 461cb3a3a11c0b4530312f22b8d5b5a19701f307 Mon Sep 17 00:00:00 2001 From: Ochenstarik Date: Thu, 30 Jul 2026 13:47:07 +0700 Subject: [PATCH] Execute confirmed timezone provisioning safely --- deploy/ochenstarik-server-monitor-manager.sh | 1 + ...chenstarik-smm-provisioning-helper.service | 2 + docs/roadmap.md | 3 +- src/ServerMonitorManager.Agent/AgentClient.cs | 100 +++++ .../ProvisioningHelperClient.cs | 47 ++ .../CertificateAuthority.cs | 17 +- .../Properties/AssemblyInfo.cs | 3 + .../ProvisioningBaseInstallPlanStore.cs | 13 +- .../ProvisioningStore.cs | 17 +- src/ServerMonitorManager.Core/Contracts.cs | 39 +- .../ProvisioningExecutionGrantCodec.cs | 3 +- .../SmmJsonContext.cs | 4 + .../Program.cs | 21 +- .../ProvisioningHelperServer.cs | 70 ++- .../TimezoneProvisioningExecutor.cs | 345 ++++++++++++++ .../CertificateAuthorityTests.cs | 13 +- .../ControlApiTests.cs | 21 +- .../ControlStoreTests.cs | 42 +- .../ProvisioningHelperTests.cs | 419 ++++++++++++++++++ tests/bootstrap/test-bootstrap-contract.sh | 5 + 20 files changed, 1138 insertions(+), 47 deletions(-) create mode 100644 src/ServerMonitorManager.Control/Properties/AssemblyInfo.cs create mode 100644 src/ServerMonitorManager.Provisioning.Helper/TimezoneProvisioningExecutor.cs diff --git a/deploy/ochenstarik-server-monitor-manager.sh b/deploy/ochenstarik-server-monitor-manager.sh index 34825df..a766e64 100755 --- a/deploy/ochenstarik-server-monitor-manager.sh +++ b/deploy/ochenstarik-server-monitor-manager.sh @@ -541,6 +541,7 @@ install_agent() { ensure_system_user "$AGENT_USER" install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR" install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent" + install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback" install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root" install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root" install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")" diff --git a/deploy/ochenstarik-smm-provisioning-helper.service b/deploy/ochenstarik-smm-provisioning-helper.service index ecf3697..d05a4b8 100644 --- a/deploy/ochenstarik-smm-provisioning-helper.service +++ b/deploy/ochenstarik-smm-provisioning-helper.service @@ -7,6 +7,7 @@ Type=simple User=root Group=ochenstarik-smm-agent UMask=0007 +EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env RuntimeDirectory=ochenstarik-server-monitor-manager RuntimeDirectoryMode=0750 WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper @@ -18,6 +19,7 @@ PrivateTmp=true PrivateDevices=true ProtectSystem=strict ProtectHome=true +ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback ProtectKernelTunables=true ProtectKernelModules=true ProtectControlGroups=true diff --git a/docs/roadmap.md b/docs/roadmap.md index 4fe63e5..b6bf159 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -112,11 +112,12 @@ - [x] state machine, confirmations, cancellation, retry и rollback; - [x] создание, чтение, подтверждение и отмена через Operator API; - [ ] выполнение, retry, verification и rollback в полной state machine; +- [x] безопасный частичный execution increment для `system.base-install`: timezone-only, factual verification и автоматический rollback; - [x] обязательные idempotency key, audit reason и job TTL; - [x] атомарный Agent job channel только для собственного `node_id`; - [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`; - [ ] versioned JSON schemas для остальных action type; -- [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`); +- [x] restricted root helper через Unix socket (`preflight`, plan и подтверждённый timezone-only execution для `system.base-install`); - [x] двухфазный `system.base-install`: сохранённый проверенный plan до Operator confirmation; - [x] короткоживущий ECDSA execution grant, привязанный к Node, job и SHA-256 подтверждённого plan; - [x] structured redacted events, bounded Operator history и progress; diff --git a/src/ServerMonitorManager.Agent/AgentClient.cs b/src/ServerMonitorManager.Agent/AgentClient.cs index 0a0e1d2..497d0df 100644 --- a/src/ServerMonitorManager.Agent/AgentClient.cs +++ b/src/ServerMonitorManager.Agent/AgentClient.cs @@ -165,6 +165,13 @@ internal sealed class AgentClient(AgentOptions options) return; } + if (job.ActionType == "system.base-install" + && job.State == ProvisioningJobStates.Running) + { + await ExecuteBaseInstallAsync(client, job, cancellationToken); + return; + } + if (job.ActionType != "preflight") { await ReportProvisioningAsync( @@ -203,6 +210,99 @@ internal sealed class AgentClient(AgentOptions options) + $"{result.OperatingSystemVersion} {result.Architecture}."); } + private async Task ExecuteBaseInstallAsync( + HttpClient client, + ProvisioningJob job, + CancellationToken cancellationToken) + { + ProvisioningBaseInstallExecutionAuthorization authorization; + try + { + authorization = await RequestBaseInstallExecutionAuthorizationAsync( + client, job, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, + "authorize", "base-install.authorization-failed", + "Base installation authorization failed before mutation.", cancellationToken); + Console.Error.WriteLine($"Base installation {job.Id} authorization failed: {exception.Message}"); + return; + } + + ProvisioningBaseInstallExecutionResult result; + try + { + var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath); + result = await helper.ExecuteBaseInstallAsync(job, authorization, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.NeedsReconciliation, job.ProgressPercent, + "reconcile", "base-install.execution-uncertain", + "Base installation outcome is uncertain and requires reconciliation.", cancellationToken); + Console.Error.WriteLine($"Base installation {job.Id} outcome is uncertain: {exception.Message}"); + return; + } + + if (result.Success) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Verifying, 90, + "verify", "base-install.verifying", "Verifying applied configuration.", cancellationToken); + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Completed, 100, + "completed", "base-install.completed", "Base installation change verified.", cancellationToken); + Console.WriteLine($"Base installation {job.Id} completed and verified."); + return; + } + + if (!result.RollbackAttempted) + { + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.Failed, job.ProgressPercent, + "execute", result.Code, "Base installation was rejected before mutation.", cancellationToken); + return; + } + + await ReportProvisioningAsync( + client, job, ProvisioningJobStates.RollingBack, 90, + "rollback", "base-install.rollback", "Restoring the previous configuration.", cancellationToken); + await ReportProvisioningAsync( + client, job, + result.RollbackSucceeded ? ProvisioningJobStates.RolledBack : ProvisioningJobStates.RollbackFailed, + 100, + result.RollbackSucceeded ? "rolled-back" : "rollback-failed", + result.RollbackSucceeded ? "base-install.rolled-back" : "base-install.rollback-failed", + result.RollbackSucceeded + ? "Previous configuration restored and verified." + : "Previous configuration could not be verified after rollback.", + cancellationToken); + } + + private static async Task + RequestBaseInstallExecutionAuthorizationAsync( + HttpClient client, + ProvisioningJob job, + CancellationToken cancellationToken) + { + var request = new ProvisioningExecutionGrantRequest( + CreateOperationId(job.Id, "execution-grant")); + using var response = await client.PostAsJsonAsync( + $"api/v1/agents/provisioning/jobs/{job.Id}/execution-grant", + request, + SmmJsonContext.Default.ProvisioningExecutionGrantRequest, + cancellationToken); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync( + SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization, + cancellationToken) + ?? throw new InvalidOperationException( + "Control service returned an empty execution authorization."); + } + private static async Task ReportProvisioningAsync( HttpClient client, ProvisioningJob job, diff --git a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs index d0c2673..f0315bc 100644 --- a/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs +++ b/src/ServerMonitorManager.Agent/ProvisioningHelperClient.cs @@ -39,6 +39,53 @@ public sealed class ProvisioningHelperClient(string socketPath) return response.BaseInstallPlan; } + public async Task ExecuteBaseInstallAsync( + ProvisioningJob job, + ProvisioningBaseInstallExecutionAuthorization authorization, + CancellationToken cancellationToken) + { + var response = await SendAsync( + CreateBaseInstallExecutionRequest(job, authorization), cancellationToken); + return ValidateBaseInstallExecutionResponse(response, authorization); + } + + internal static ProvisioningBaseInstallExecutionResult ValidateBaseInstallExecutionResponse( + ProvisioningHelperResponse response, + ProvisioningBaseInstallExecutionAuthorization authorization) + { + if (response.BaseInstallExecution is null) + { + throw new InvalidOperationException( + $"Provisioning helper returned no execution result: {response.Code}"); + } + var result = response.BaseInstallExecution; + if (response.Success != result.Success + || (result.Success && (!result.Verified + || result.RollbackAttempted + || result.RollbackSucceeded + || !string.Equals( + result.ObservedTimezone, + authorization.Plan.Timezone, + StringComparison.Ordinal))) + || (result.RollbackSucceeded && !result.RollbackAttempted)) + { + throw new InvalidDataException("Provisioning helper returned an inconsistent execution result."); + } + return result; + } + + internal static ProvisioningHelperRequest CreateBaseInstallExecutionRequest( + ProvisioningJob job, + ProvisioningBaseInstallExecutionAuthorization authorization) + => new( + ProvisioningExecutionGrantCodec.ProtocolVersion, + job.Id, + job.ActionType, + job.SchemaVersion, + ProvisioningActionCatalog.SystemBaseInstallModuleHash, + job.Parameters, + authorization); + private async Task SendAsync( ProvisioningHelperRequest request, CancellationToken cancellationToken) diff --git a/src/ServerMonitorManager.Control/CertificateAuthority.cs b/src/ServerMonitorManager.Control/CertificateAuthority.cs index 91b0796..a70e7fa 100644 --- a/src/ServerMonitorManager.Control/CertificateAuthority.cs +++ b/src/ServerMonitorManager.Control/CertificateAuthority.cs @@ -74,16 +74,25 @@ public sealed class CertificateAuthority : IDisposable { if (job.ActionType != "system.base-install" || job.SchemaVersion != 1 - || job.State != ProvisioningJobStates.Queued + || job.State != ProvisioningJobStates.Running || !job.ConfirmationRequired || job.ConfirmedAt is null || lifetime <= TimeSpan.Zero || lifetime > ProvisioningExecutionGrantCodec.MaximumLifetime) { throw new InvalidOperationException( - "Only a confirmed queued base installation job can receive an execution grant."); + "Only a confirmed running base installation job can receive an execution grant."); } + var expiresAt = issuedAt.Add(lifetime) < job.ExpiresAt + ? issuedAt.Add(lifetime) + : job.ExpiresAt; + var issuedAtUnixSeconds = issuedAt.ToUnixTimeSeconds(); + var expiresAtUnixSeconds = expiresAt.ToUnixTimeSeconds(); + if (expiresAtUnixSeconds <= issuedAtUnixSeconds) + { + throw new InvalidOperationException("Provisioning job expires before a grant can be issued."); + } var grant = new ProvisioningExecutionGrant( ProvisioningExecutionGrantCodec.ProtocolVersion, job.Id, @@ -91,8 +100,8 @@ public sealed class CertificateAuthority : IDisposable job.ActionType, job.SchemaVersion, ProvisioningExecutionGrantCodec.ComputePlanSha256(plan), - issuedAt.ToUnixTimeSeconds(), - issuedAt.Add(lifetime).ToUnixTimeSeconds(), + issuedAtUnixSeconds, + expiresAtUnixSeconds, Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)), ProvisioningExecutionGrantCodec.SignatureAlgorithm, string.Empty); diff --git a/src/ServerMonitorManager.Control/Properties/AssemblyInfo.cs b/src/ServerMonitorManager.Control/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..f4c17bd --- /dev/null +++ b/src/ServerMonitorManager.Control/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ServerMonitorManager.Control.Tests")] diff --git a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs index 8abbe0b..074581f 100644 --- a/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningBaseInstallPlanStore.cs @@ -133,7 +133,7 @@ public sealed partial class ControlStore DateTimeOffset.Parse(reader.GetString(2))); } - public async Task IssueBaseInstallExecutionGrantAsync( + public async Task IssueBaseInstallExecutionGrantAsync( string nodeId, string jobId, ProvisioningExecutionGrantRequest request, @@ -148,7 +148,7 @@ public sealed partial class ControlStore request, SmmJsonContext.Default.ProvisioningExecutionGrantRequest); var cached = await ReadIdempotentAsync( connection, transaction, operationKey, requestHash, - SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken); + SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization, cancellationToken); if (cached is not null) { await transaction.CommitAsync(cancellationToken); @@ -163,7 +163,7 @@ public sealed partial class ControlStore } if (job.ActionType != "system.base-install" || job.SchemaVersion != 1 - || job.State != ProvisioningJobStates.Queued + || job.State != ProvisioningJobStates.Running || !job.ConfirmationRequired || job.ConfirmedAt is null || job.ExpiresAt <= DateTimeOffset.UtcNow) @@ -187,9 +187,10 @@ public sealed partial class ControlStore planJson, SmmJsonContext.Default.SystemBaseInstallPlan) ?? throw new InvalidDataException("Stored base installation plan is invalid."); var grant = issueGrant(job, plan); + var authorization = new ProvisioningBaseInstallExecutionAuthorization(nodeId, plan, grant); await WriteIdempotentAsync( - connection, transaction, operationKey, requestHash, grant, - SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken); + connection, transaction, operationKey, requestHash, authorization, + SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization, cancellationToken); await WriteAuditAsync( connection, transaction, nodeId, "provisioning.execution-grant.issued", jobId, JsonSerializer.Serialize(new @@ -201,7 +202,7 @@ public sealed partial class ControlStore }), cancellationToken); await transaction.CommitAsync(cancellationToken); - return grant; + return authorization; } } diff --git a/src/ServerMonitorManager.Control/ProvisioningStore.cs b/src/ServerMonitorManager.Control/ProvisioningStore.cs index 8cb5b30..1086203 100644 --- a/src/ServerMonitorManager.Control/ProvisioningStore.cs +++ b/src/ServerMonitorManager.Control/ProvisioningStore.cs @@ -107,8 +107,13 @@ public sealed partial class ControlStore command.Transaction = transaction; command.CommandText = """ UPDATE provisioning_jobs SET - state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END, + state = CASE + WHEN state = $queued AND confirmed_at IS NOT NULL THEN $running + WHEN state = $queued THEN $preflight + ELSE $rolling_back + END, current_step = CASE + WHEN state = $queued AND confirmed_at IS NOT NULL THEN 'execute' WHEN state = $queued THEN 'preflight' ELSE 'rollback' END, @@ -117,22 +122,22 @@ public sealed partial class ControlStore WHERE id = ( SELECT id FROM provisioning_jobs WHERE node_id = $node - AND ((state = $queued - AND (confirmation_required = 0 OR confirmed_at IS NULL)) + AND (state = $queued OR (state = $rolling_back AND current_step = 'rollback-queued')) AND expires_at > $now ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id LIMIT 1) - AND ((state = $queued - AND (confirmation_required = 0 OR confirmed_at IS NULL)) + AND (state = $queued OR (state = $rolling_back AND current_step = 'rollback-queued')) RETURNING *; """; command.Parameters.AddWithValue("$node", nodeId); command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued); command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight); + command.Parameters.AddWithValue("$running", ProvisioningJobStates.Running); command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack); command.Parameters.AddWithValue("$now", now.ToString("O")); + ProvisioningJob? job; await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { @@ -656,7 +661,9 @@ internal static class ProvisioningStateMachine (ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true, (ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true, (ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Running, ProvisioningJobStates.RollingBack) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true, + (ProvisioningJobStates.Verifying, ProvisioningJobStates.RollingBack) => true, (ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100, (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true, (ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100, diff --git a/src/ServerMonitorManager.Core/Contracts.cs b/src/ServerMonitorManager.Core/Contracts.cs index c0456db..b4deb42 100644 --- a/src/ServerMonitorManager.Core/Contracts.cs +++ b/src/ServerMonitorManager.Core/Contracts.cs @@ -171,14 +171,16 @@ public sealed record ProvisioningHelperRequest( string ActionType, int SchemaVersion, string ModuleHash, - JsonElement Parameters); + JsonElement Parameters, + ProvisioningBaseInstallExecutionAuthorization? Execution = null); public sealed record ProvisioningHelperResponse( bool Success, string Code, string Message, ProvisioningPreflightResult? Preflight, - SystemBaseInstallPlan? BaseInstallPlan); + SystemBaseInstallPlan? BaseInstallPlan, + ProvisioningBaseInstallExecutionResult? BaseInstallExecution = null); [JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] public sealed record ProvisioningPreflightResult( @@ -301,6 +303,37 @@ public sealed record ProvisioningExecutionGrant( [JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] public sealed record ProvisioningExecutionGrantRequest(string IdempotencyKey); +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record ProvisioningBaseInstallExecutionAuthorization( + string NodeId, + SystemBaseInstallPlan Plan, + ProvisioningExecutionGrant Grant); + +public sealed record ProvisioningBaseInstallExecutionResult( + bool Success, + string Code, + string Message, + bool Changed, + bool Verified, + bool RollbackAttempted, + bool RollbackSucceeded, + string? ObservedTimezone); + +public sealed record ProvisioningTimezoneRollbackRecord( + int SchemaVersion, + string JobId, + string NodeId, + string PlanSha256, + string PreviousTimezone, + string RequestedTimezone); + +public sealed record ProvisioningExecutionConsumptionRecord( + int SchemaVersion, + string JobId, + string NodeId, + string Nonce, + DateTimeOffset ConsumedAt); + public static class PreflightDriftStatuses { public const string NotConfigured = "NotConfigured"; @@ -430,7 +463,7 @@ public static class SystemBaseInstallSchema && parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count() == parameters.PackageGroupIds.Length && parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup) - && parameters.SwapMode is "disabled" or "automatic" or "explicit" + && parameters.SwapMode is "unchanged" or "disabled" or "automatic" or "explicit" && (parameters.SwapMode == "explicit" ? parameters.SwapSizeMiB is >= 128 and <= 1_048_576 : parameters.SwapSizeMiB is null) diff --git a/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs index e97e9eb..2c65c84 100644 --- a/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs +++ b/src/ServerMonitorManager.Core/ProvisioningExecutionGrantCodec.cs @@ -95,7 +95,8 @@ public static class ProvisioningExecutionGrantCodec { return false; } - if (signature.Length != 64) + if (signature.Length != 64 + || !string.Equals(EncodeBase64Url(signature), grant.Signature, StringComparison.Ordinal)) { return false; } diff --git a/src/ServerMonitorManager.Core/SmmJsonContext.cs b/src/ServerMonitorManager.Core/SmmJsonContext.cs index 88e6a38..97235a1 100644 --- a/src/ServerMonitorManager.Core/SmmJsonContext.cs +++ b/src/ServerMonitorManager.Core/SmmJsonContext.cs @@ -47,6 +47,10 @@ namespace ServerMonitorManager.Core; [JsonSerializable(typeof(ProvisioningBaseInstallPlanRecord))] [JsonSerializable(typeof(ProvisioningExecutionGrant))] [JsonSerializable(typeof(ProvisioningExecutionGrantRequest))] +[JsonSerializable(typeof(ProvisioningBaseInstallExecutionAuthorization))] +[JsonSerializable(typeof(ProvisioningBaseInstallExecutionResult))] +[JsonSerializable(typeof(ProvisioningTimezoneRollbackRecord))] +[JsonSerializable(typeof(ProvisioningExecutionConsumptionRecord))] [JsonSerializable(typeof(ProvisioningJob))] [JsonSerializable(typeof(ProvisioningJob[]))] [JsonSerializable(typeof(ProvisioningEvent))] diff --git a/src/ServerMonitorManager.Provisioning.Helper/Program.cs b/src/ServerMonitorManager.Provisioning.Helper/Program.cs index 0407ecc..9ebef5a 100644 --- a/src/ServerMonitorManager.Provisioning.Helper/Program.cs +++ b/src/ServerMonitorManager.Provisioning.Helper/Program.cs @@ -1,3 +1,4 @@ +using System.Security.Cryptography.X509Certificates; using ServerMonitorManager.Provisioning.Helper; if (!OperatingSystem.IsLinux()) @@ -12,6 +13,16 @@ if (args.Length != 0) return 2; } const string socketPath = "/run/ochenstarik-server-monitor-manager/provisioning.sock"; +const string controlAuthorityPath = "/etc/ochenstarik-server-monitor-manager/control-ca.crt"; +const string rollbackDirectory = + "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"; +var localNodeId = Environment.GetEnvironmentVariable("SMM_NodeId"); +if (localNodeId is not { Length: >= 1 and <= 63 } + || !localNodeId.All(character => character is >= 'a' and <= 'z' or >= '0' and <= '9' or '-')) +{ + Console.Error.WriteLine("SMM_NodeId must identify the local enrolled Node."); + return 2; +} using var shutdown = new CancellationTokenSource(); Console.CancelKeyPress += (_, eventArgs) => @@ -20,5 +31,13 @@ Console.CancelKeyPress += (_, eventArgs) => shutdown.Cancel(); }; -await new ProvisioningHelperServer(socketPath).RunAsync(shutdown.Token); +using var controlAuthority = X509CertificateLoader.LoadCertificateFromFile(controlAuthorityPath); +var timezoneExecutor = new TimezoneProvisioningExecutor( + controlAuthority, + localNodeId, + new ProvisioningFileSystem(), + new ProvisioningProcessRunner(), + TimeProvider.System, + rollbackDirectory); +await new ProvisioningHelperServer(socketPath, timezoneExecutor).RunAsync(shutdown.Token); return 0; diff --git a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs index b1c0455..210ddc0 100644 --- a/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs +++ b/src/ServerMonitorManager.Provisioning.Helper/ProvisioningHelperServer.cs @@ -7,18 +7,28 @@ using ServerMonitorManager.Core; namespace ServerMonitorManager.Provisioning.Helper; -public sealed class ProvisioningHelperServer(string socketPath) +public sealed class ProvisioningHelperServer { private const int MaximumRequestBytes = 16 * 1024; + private readonly string _socketPath; + private readonly TimezoneProvisioningExecutor? _timezoneExecutor; + + public ProvisioningHelperServer( + string socketPath, + TimezoneProvisioningExecutor? timezoneExecutor = null) + { + _socketPath = socketPath; + _timezoneExecutor = timezoneExecutor; + } [SupportedOSPlatform("linux")] public async Task RunAsync(CancellationToken cancellationToken) { - Directory.CreateDirectory(Path.GetDirectoryName(socketPath)!); - File.Delete(socketPath); + Directory.CreateDirectory(Path.GetDirectoryName(_socketPath)!); + File.Delete(_socketPath); using var listener = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); - listener.Bind(new UnixDomainSocketEndPoint(socketPath)); - File.SetUnixFileMode(socketPath, + listener.Bind(new UnixDomainSocketEndPoint(_socketPath)); + File.SetUnixFileMode(_socketPath, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.GroupWrite); listener.Listen(8); @@ -27,22 +37,39 @@ public sealed class ProvisioningHelperServer(string socketPath) while (!cancellationToken.IsCancellationRequested) { var connection = await listener.AcceptAsync(cancellationToken); - await HandleAsync(connection, cancellationToken); + try + { + await HandleAsync(connection, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + connection.Dispose(); + // A malformed or disconnected local client must not terminate the helper listener. + } } } finally { - File.Delete(socketPath); + File.Delete(_socketPath); } } public static ProvisioningHelperResponse Execute(ProvisioningHelperRequest request) + => Execute(request, null); + + public static ProvisioningHelperResponse Execute( + ProvisioningHelperRequest request, + TimezoneProvisioningExecutor? timezoneExecutor) { if (request.ProtocolVersion != "1") { return Failure("protocol.unsupported", "Unsupported helper protocol version."); } - if (request.JobId.Length != 32 || !request.JobId.All(Uri.IsHexDigit)) + if (request.JobId is not { Length: 32 } || !request.JobId.All(Uri.IsHexDigit)) { return Failure("request.invalid-job", "Invalid provisioning job identifier."); } @@ -54,11 +81,26 @@ public sealed class ProvisioningHelperServer(string socketPath) return request.ActionType switch { "preflight" => ExecutePreflight(request), - "system.base-install" => CreateBaseInstallPlan(request), + "system.base-install" => request.Execution is null + ? CreateBaseInstallPlan(request) + : ExecuteBaseInstall(request, timezoneExecutor), _ => Failure("action.denied", "The requested action is not allowed.") }; } + private static ProvisioningHelperResponse ExecuteBaseInstall( + ProvisioningHelperRequest request, + TimezoneProvisioningExecutor? timezoneExecutor) + { + if (timezoneExecutor is null) + { + return Failure("execution.unavailable", "Provisioning execution is unavailable."); + } + var result = timezoneExecutor.Execute(request); + return new ProvisioningHelperResponse( + result.Success, result.Code, result.Message, null, null, result); + } + private static ProvisioningHelperResponse ExecutePreflight(ProvisioningHelperRequest request) { if (request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash @@ -114,7 +156,7 @@ public sealed class ProvisioningHelperServer(string socketPath) true, "system.base-install.plan-ready", "Base install plan is ready.", null, plan); } - private static async Task HandleAsync(Socket socket, CancellationToken cancellationToken) + private async Task HandleAsync(Socket socket, CancellationToken cancellationToken) { using (socket) await using (var stream = new NetworkStream(socket, ownsSocket: false)) @@ -125,9 +167,13 @@ public sealed class ProvisioningHelperServer(string socketPath) var payload = await ReadRequestAsync(stream, cancellationToken); var request = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperRequest) ?? throw new JsonException("Empty request."); - response = Execute(request); + response = Execute(request, _timezoneExecutor); } - catch (Exception exception) when (exception is JsonException or InvalidDataException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception) { response = Failure("request.invalid", "Invalid helper request."); } diff --git a/src/ServerMonitorManager.Provisioning.Helper/TimezoneProvisioningExecutor.cs b/src/ServerMonitorManager.Provisioning.Helper/TimezoneProvisioningExecutor.cs new file mode 100644 index 0000000..b6e1e4e --- /dev/null +++ b/src/ServerMonitorManager.Provisioning.Helper/TimezoneProvisioningExecutor.cs @@ -0,0 +1,345 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using ServerMonitorManager.Core; + +namespace ServerMonitorManager.Provisioning.Helper; + +public interface IProvisioningFileSystem +{ + bool FileExists(string path); + bool IsSymbolicLink(string path); + void CreateOwnerOnlyDirectory(string path); + void WriteOwnerOnlyFile(string path, string content); +} + +public interface IProvisioningProcessRunner +{ + ProvisioningProcessResult Run(string fileName, IReadOnlyList arguments); +} + +public sealed record ProvisioningProcessResult(int ExitCode, string StandardOutput, string StandardError); + +public sealed class TimezoneProvisioningExecutor( + X509Certificate2 controlAuthority, + string localNodeId, + IProvisioningFileSystem fileSystem, + IProvisioningProcessRunner processRunner, + TimeProvider timeProvider, + string rollbackDirectory) +{ + private const string TimedatectlPath = "/usr/bin/timedatectl"; + private const string ZoneinfoRoot = "/usr/share/zoneinfo"; + + public ProvisioningBaseInstallExecutionResult Execute(ProvisioningHelperRequest request) + { + var authorization = request.Execution; + if (authorization is null + || request.ProtocolVersion != ProvisioningExecutionGrantCodec.ProtocolVersion + || request.JobId is not { Length: 32 } + || !request.JobId.All(Uri.IsHexDigit) + || request.ActionType != "system.base-install" + || request.SchemaVersion != 1 + || request.ModuleHash != ProvisioningActionCatalog.SystemBaseInstallModuleHash + || !string.Equals(authorization.NodeId, localNodeId, StringComparison.Ordinal) + || !ProvisioningExecutionGrantCodec.Verify( + authorization.Grant, + controlAuthority, + request.JobId, + localNodeId, + authorization.Plan, + timeProvider.GetUtcNow())) + { + return Failure("execution.authorization-denied", "Execution authorization was rejected."); + } + + var plan = authorization.Plan; + if (!IsTimezoneOnly(plan)) + { + return Failure( + "system.base-install.unsupported-fields", + "Only timezone changes are supported by this execution increment."); + } + if (!IsSafeTimezone(plan.Timezone) + || !fileSystem.FileExists($"{ZoneinfoRoot}/{plan.Timezone}")) + { + return Failure("timezone.invalid", "The requested timezone is unavailable."); + } + if (!rollbackDirectory.StartsWith("/", StringComparison.Ordinal) + || HasSymlinkedManagedPath(rollbackDirectory)) + { + return Failure("backup.unsafe-path", "The rollback path is unsafe."); + } + + try + { + fileSystem.CreateOwnerOnlyDirectory(rollbackDirectory); + var nonceHash = Convert.ToHexStringLower( + SHA256.HashData(Encoding.UTF8.GetBytes(authorization.Grant.Nonce))); + var consumptionPath = $"{rollbackDirectory}/grant-{nonceHash}.consumed.json"; + if (fileSystem.IsSymbolicLink(consumptionPath)) + { + return Failure("backup.unsafe-path", "The grant consumption path is unsafe."); + } + fileSystem.WriteOwnerOnlyFile( + consumptionPath, + JsonSerializer.Serialize( + new ProvisioningExecutionConsumptionRecord( + 1, + request.JobId, + localNodeId, + authorization.Grant.Nonce, + timeProvider.GetUtcNow()), + SmmJsonContext.Default.ProvisioningExecutionConsumptionRecord)); + } + catch + { + return Failure( + "execution.grant-consumed", + "Execution grant was already consumed or could not be recorded."); + } + + ProvisioningProcessResult observedBefore; + try + { + observedBefore = QueryTimezone(); + } + catch + { + return Failure("timezone.inspect-failed", "The current timezone could not be inspected."); + } + var previousTimezone = NormalizeTimezone(observedBefore); + if (previousTimezone is null || !IsSafeTimezone(previousTimezone)) + { + return Failure("timezone.inspect-failed", "The current timezone could not be inspected."); + } + if (string.Equals(previousTimezone, plan.Timezone, StringComparison.Ordinal)) + { + return new ProvisioningBaseInstallExecutionResult( + true, "timezone.already-configured", "Timezone is already configured.", + false, true, false, false, previousTimezone); + } + + try + { + var rollbackPath = $"{rollbackDirectory}/{request.JobId}.json"; + if (fileSystem.IsSymbolicLink(rollbackPath)) + { + return Failure("backup.unsafe-path", "The rollback path is unsafe."); + } + fileSystem.WriteOwnerOnlyFile( + rollbackPath, + CreateRollbackRecord(request, authorization, previousTimezone)); + } + catch + { + return Failure("backup.create-failed", "The rollback record could not be created."); + } + + var failureCode = "timezone.mutation-failed"; + try + { + var mutation = SetTimezone(plan.Timezone); + if (mutation.ExitCode == 0) + { + failureCode = "timezone.verification-failed"; + var observedAfter = NormalizeTimezone(QueryTimezone()); + if (string.Equals(observedAfter, plan.Timezone, StringComparison.Ordinal)) + { + return new ProvisioningBaseInstallExecutionResult( + true, "timezone.changed", "Timezone was changed and verified.", + true, true, false, false, observedAfter); + } + } + } + catch + { + // A failed process or verification can still have partially changed the host. + } + + var rollbackSucceeded = false; + string? observedRollback = null; + try + { + var rollback = SetTimezone(previousTimezone); + if (rollback.ExitCode == 0) + { + observedRollback = NormalizeTimezone(QueryTimezone()); + rollbackSucceeded = string.Equals( + observedRollback, previousTimezone, StringComparison.Ordinal); + } + } + catch + { + rollbackSucceeded = false; + } + + return new ProvisioningBaseInstallExecutionResult( + false, + failureCode, + failureCode == "timezone.verification-failed" + ? "Timezone verification failed; rollback was attempted." + : "Timezone mutation failed; rollback was attempted.", + false, + false, + true, + rollbackSucceeded, + observedRollback); + } + + private ProvisioningProcessResult QueryTimezone() + => processRunner.Run( + TimedatectlPath, + ["show", "--property=Timezone", "--value"]); + + private ProvisioningProcessResult SetTimezone(string timezone) + => processRunner.Run(TimedatectlPath, ["set-timezone", timezone]); + + private bool HasSymlinkedManagedPath(string path) + { + var current = string.Empty; + foreach (var component in path.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + current += $"/{component}"; + if (fileSystem.IsSymbolicLink(current)) + { + return true; + } + } + return false; + } + + private static string CreateRollbackRecord( + ProvisioningHelperRequest request, + ProvisioningBaseInstallExecutionAuthorization authorization, + string previousTimezone) + => JsonSerializer.Serialize( + new ProvisioningTimezoneRollbackRecord( + 1, + request.JobId, + authorization.NodeId, + authorization.Grant.PlanSha256, + previousTimezone, + authorization.Plan.Timezone), + SmmJsonContext.Default.ProvisioningTimezoneRollbackRecord) + + "\n"; + + private static bool IsTimezoneOnly(SystemBaseInstallPlan plan) + => string.Equals(plan.Locale, "unchanged", StringComparison.Ordinal) + && !plan.AptUpdate + && !plan.AptUpgrade + && plan.Packages is { Length: 0 } + && string.Equals(plan.SwapMode, "unchanged", StringComparison.Ordinal) + && plan.SwapSizeMiB is null + && plan.VmSwappiness == 60 + && !plan.EnableUnattendedUpgrades + && string.Equals(plan.RebootPolicy, "never", StringComparison.Ordinal); + + private static bool IsSafeTimezone(string? value) + => value is { Length: >= 1 and <= 64 } + && value[0] is not '/' and not '.' + && !value.Contains("..", StringComparison.Ordinal) + && value.All(character => char.IsAsciiLetterOrDigit(character) + || character is '/' or '_' or '-' or '+'); + + private static string? NormalizeTimezone(ProvisioningProcessResult result) + { + if (result.ExitCode != 0) + { + return null; + } + var value = result.StandardOutput.Trim(); + return value.Length == 0 || value.Contains('\n') || value.Contains('\r') + ? null + : value; + } + + private static ProvisioningBaseInstallExecutionResult Failure(string code, string message) + => new(false, code, message, false, false, false, false, null); +} + +public sealed class ProvisioningFileSystem : IProvisioningFileSystem +{ + public bool FileExists(string path) => File.Exists(path); + + public bool IsSymbolicLink(string path) + { + var info = File.Exists(path) + ? (FileSystemInfo)new FileInfo(path) + : Directory.Exists(path) + ? new DirectoryInfo(path) + : new FileInfo(path); + return info.Exists && info.LinkTarget is not null; + } + + public void CreateOwnerOnlyDirectory(string path) + { + Directory.CreateDirectory(path); + if (OperatingSystem.IsLinux()) + { + File.SetUnixFileMode(path, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + + public void WriteOwnerOnlyFile(string path, string content) + { + var options = new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = 4096, + Options = FileOptions.WriteThrough + }; + if (OperatingSystem.IsLinux()) + { + options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + } + using var stream = new FileStream(path, options); + using var writer = new StreamWriter(stream, new UTF8Encoding(false)); + writer.Write(content); + writer.Flush(); + stream.Flush(flushToDisk: true); + } +} + +public sealed class ProvisioningProcessRunner : IProvisioningProcessRunner +{ + public ProvisioningProcessResult Run(string fileName, IReadOnlyList arguments) + { + if (!string.Equals(fileName, "/usr/bin/timedatectl", StringComparison.Ordinal)) + { + throw new InvalidOperationException("Executable is outside the provisioning allowlist."); + } + var startInfo = new ProcessStartInfo + { + FileName = fileName, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Provisioning process could not be started."); + var standardOutput = process.StandardOutput.ReadToEndAsync(); + var standardError = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(); + throw new TimeoutException("Provisioning process timed out."); + } + Task.WaitAll(standardOutput, standardError); + return new ProvisioningProcessResult( + process.ExitCode, + standardOutput.GetAwaiter().GetResult(), + standardError.GetAwaiter().GetResult()); + } +} diff --git a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs index 210f2ac..522c013 100644 --- a/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/CertificateAuthorityTests.cs @@ -79,9 +79,9 @@ public sealed class CertificateAuthorityTests : IDisposable SmmJsonContext.Default.SystemBaseInstallParameters); var job = new ProvisioningJob( Guid.NewGuid().ToString("N"), "home", "system.base-install", 1, parameters, - ProvisioningJobStates.Queued, true, "test", "operator", + ProvisioningJobStates.Running, true, "test", "operator", now.AddMinutes(-1), now, now.AddMinutes(30), now, null, 4, 25, - "confirmed-queued", null); + "execute", null); var plan = new SystemBaseInstallPlan( "UTC", "en_US.UTF-8", true, false, SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]), @@ -99,6 +99,15 @@ public sealed class CertificateAuthorityTests : IDisposable plan with { Packages = [.. plan.Packages, "untrusted-package"] }, now)); Assert.False(ProvisioningExecutionGrantCodec.Verify( grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now.AddMinutes(3))); + var expiringJob = job with { ExpiresAt = now.AddSeconds(15) }; + var clamped = authority.SignProvisioningExecutionGrant( + expiringJob, plan, now, TimeSpan.FromMinutes(2)); + Assert.Equal(expiringJob.ExpiresAt.ToUnixTimeSeconds(), clamped.ExpiresAtUnixSeconds); + Assert.Throws(() => authority.SignProvisioningExecutionGrant( + job with { ExpiresAt = now }, plan, now, TimeSpan.FromMinutes(2))); + Assert.Throws(() => authority.SignProvisioningExecutionGrant( + job with { ExpiresAt = now.AddMilliseconds(100) }, + plan, now, TimeSpan.FromMinutes(2))); Assert.Throws(() => authority.SignProvisioningExecutionGrant( job with { ConfirmedAt = null }, plan, now, TimeSpan.FromMinutes(2))); } diff --git a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs index 0c9531a..defbeb8 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlApiTests.cs @@ -181,29 +181,36 @@ public sealed class ControlApiTests : IAsyncDisposable ServerMonitorManager.Core.ProvisioningJob>(cancellationToken); Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, confirmed!.State); Assert.Equal("confirmed-queued", confirmed.CurrentStep); + var execution = await agent.GetFromJsonAsync( + "/api/v1/agents/provisioning/jobs/next", cancellationToken); + Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Running, execution!.State); var grantRequest = new { idempotencyKey = Guid.NewGuid().ToString() }; var grantResponse = await agent.PostAsJsonAsync( $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant", grantRequest, cancellationToken); Assert.Equal(HttpStatusCode.OK, grantResponse.StatusCode); - var grant = await grantResponse.Content.ReadFromJsonAsync< - ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken); + var authorization = await grantResponse.Content.ReadFromJsonAsync< + ServerMonitorManager.Core.ProvisioningBaseInstallExecutionAuthorization>(cancellationToken); var grantReplayResponse = await agent.PostAsJsonAsync( $"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant", grantRequest, cancellationToken); - var grantReplay = await grantReplayResponse.Content.ReadFromJsonAsync< - ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken); - Assert.Equal(grant!.Signature, grantReplay!.Signature); + var authorizationReplay = await grantReplayResponse.Content.ReadFromJsonAsync< + ServerMonitorManager.Core.ProvisioningBaseInstallExecutionAuthorization>(cancellationToken); + Assert.Equal(authorization!.Grant.Signature, authorizationReplay!.Grant.Signature); + Assert.Equal( + ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.ComputePlanSha256(storedPlan.Plan), + ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.ComputePlanSha256(authorization.Plan)); + Assert.Equal("home", authorization.NodeId); var authority = _factory.Services.GetRequiredService< ServerMonitorManager.Control.CertificateAuthority>(); Assert.True(ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.Verify( - grant, + authorization.Grant, authority.PublicCertificate, job.Id, "home", - storedPlan.Plan, + authorization.Plan, DateTimeOffset.UtcNow)); Assert.Equal(HttpStatusCode.NoContent, (await agent.GetAsync( "/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode); diff --git a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs index d17e00e..aa78a88 100644 --- a/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ControlStoreTests.cs @@ -11,6 +11,19 @@ public sealed class ControlStoreTests : IAsyncDisposable { private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-tests-{Guid.NewGuid():N}"); + [Fact] + public void VerifiedHelperRollbackCanBeReportedToControl() + { + Assert.True(ProvisioningStateMachine.CanReport( + ProvisioningJobStates.Running, ProvisioningJobStates.RollingBack, 40, 90)); + Assert.True(ProvisioningStateMachine.CanReport( + ProvisioningJobStates.Verifying, ProvisioningJobStates.RollingBack, 90, 90)); + Assert.True(ProvisioningStateMachine.CanReport( + ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack, 90, 100)); + Assert.True(ProvisioningStateMachine.CanReport( + ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollbackFailed, 90, 100)); + } + [Fact] public async Task VersionOneDatabaseMigratesToProvisioningSchema() { @@ -99,19 +112,38 @@ public sealed class ControlStoreTests : IAsyncDisposable Assert.Equal(ProvisioningJobStates.Queued, confirmed.State); Assert.Equal("confirmed-queued", confirmed.CurrentStep); Assert.NotNull(confirmed.ConfirmedAt); + var execution = await store.ClaimNextProvisioningJobAsync("home", cancellationToken); + Assert.Equal(ProvisioningJobStates.Running, execution!.State); + Assert.Equal("execute", execution.CurrentStep); Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken)); - var cancelled = await store.CancelProvisioningJobAsync( + var rollingBack = await store.ReportProvisioningProgressAsync( + "home", created.Id, - new ProvisioningJobCommandRequest("Test completed", Guid.NewGuid().ToString()), - "operator", + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RollingBack, 90, "rollback", + "execution.rollback", "Restoring previous state.", Guid.NewGuid().ToString()), cancellationToken); - Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State); - Assert.NotNull(cancelled.CancelledAt); + Assert.Equal(ProvisioningJobStates.RollingBack, rollingBack!.State); + var rolledBack = await store.ReportProvisioningProgressAsync( + "home", + created.Id, + new ProvisioningJobProgressRequest( + ProvisioningJobStates.RolledBack, 100, "rolled-back", + "execution.rolled-back", "Previous state restored.", Guid.NewGuid().ToString()), + cancellationToken); + Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State); var next = await store.CreateProvisioningJobAsync( "home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken); Assert.Equal(ProvisioningJobStates.Queued, next.State); + var cancelled = await store.CancelProvisioningJobAsync( + next.Id, + new ProvisioningJobCommandRequest("Cancel before execution", Guid.NewGuid().ToString()), + "operator", + cancellationToken); + Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State); + Assert.NotNull(cancelled.CancelledAt); } [Fact] diff --git a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs index 2ed11c2..a518f9e 100644 --- a/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs +++ b/tests/ServerMonitorManager.Control.Tests/ProvisioningHelperTests.cs @@ -1,4 +1,7 @@ +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; using System.Text.Json; +using ServerMonitorManager.Agent; using ServerMonitorManager.Core; using ServerMonitorManager.Provisioning.Helper; using Xunit; @@ -54,6 +57,18 @@ public sealed class ProvisioningHelperTests JsonSerializer.Deserialize(json, SmmJsonContext.Default.ProvisioningHelperRequest)); } + [Fact] + public void HelperRejectsMissingJobIdWithoutThrowing() + { + using var document = JsonDocument.Parse("{}"); + var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest( + "1", null!, "preflight", 1, + ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone())); + + Assert.False(response.Success); + Assert.Equal("request.invalid-job", response.Code); + } + [Fact] public void BaseInstallSchemaRejectsCommandText() { @@ -88,4 +103,408 @@ public sealed class ProvisioningHelperTests response.BaseInstallPlan!.Packages); Assert.Equal("never", response.BaseInstallPlan.RebootPolicy); } + + [Fact] + public void ConfirmedTimezoneOnlyPlanUsesAllowlistedBinaryCreatesBackupAndVerifies() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var grant = SignGrant(authority, signingKey, plan); + var events = new List(); + var files = new FakeFileSystem(events); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + var process = new FakeProcessRunner(events, + new(0, "UTC\n", ""), + new(0, "", ""), + new(0, "Europe/Berlin\n", "")); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest(plan, grant)); + + Assert.True(result.Success); + Assert.True(result.Changed); + Assert.True(result.Verified); + Assert.False(result.RollbackAttempted); + Assert.Equal("Europe/Berlin", result.ObservedTimezone); + Assert.Equal(2, files.Writes.Count); + Assert.True(events.IndexOf("write-backup") < events.IndexOf("process:set-timezone Europe/Berlin")); + Assert.All(process.Calls, call => Assert.Equal("/usr/bin/timedatectl", call.FileName)); + Assert.Equal( + [ + "show --property=Timezone --value", + "set-timezone Europe/Berlin", + "show --property=Timezone --value" + ], + process.Calls.Select(call => string.Join(' ', call.Arguments))); + } + } + + [Fact] + public void ValidNoOpTimezoneCompletesOnlyAfterFactualVerification() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("UTC"); + var process = new FakeProcessRunner( + [], new ProvisioningProcessResult(0, "UTC\n", "")); + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/UTC"); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest(plan, SignGrant(authority, signingKey, plan))); + + Assert.True(result.Success); + Assert.False(result.Changed); + Assert.True(result.Verified); + Assert.Single(files.Writes); + Assert.Single(process.Calls); + } + } + + [Fact] + public void ConsumedGrantCannotBeReplayedEvenAfterVerifiedNoOp() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("UTC"); + var grant = SignGrant(authority, signingKey, plan); + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/UTC"); + var firstProcess = new FakeProcessRunner( + [], new ProvisioningProcessResult(0, "UTC\n", "")); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, firstProcess, TimeProvider.System, + "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var first = executor.Execute(CreateExecutionRequest(plan, grant)); + var secondProcess = new FakeProcessRunner([]); + var replayExecutor = new TimezoneProvisioningExecutor( + authority, "home", files, secondProcess, TimeProvider.System, + "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + var replay = replayExecutor.Execute(CreateExecutionRequest(plan, grant)); + + Assert.True(first.Success); + Assert.False(replay.Success); + Assert.Equal("execution.grant-consumed", replay.Code); + Assert.Empty(secondProcess.Calls); + } + } + + [Fact] + public void ForgedMismatchedAndExpiredGrantsCauseZeroMutation() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var valid = SignGrant(authority, signingKey, plan); + var invalid = new[] + { + valid with { Signature = valid.Signature[..^1] + (valid.Signature[^1] == 'A' ? "B" : "A") }, + valid with { JobId = new string('f', 32) }, + valid with { NodeId = "other-node" }, + valid with { PlanSha256 = new string('0', 64) }, + SignGrant(authority, signingKey, plan, DateTimeOffset.UtcNow.AddMinutes(-10)) + }; + + foreach (var grant in invalid) + { + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + var process = new FakeProcessRunner([]); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest(plan, grant)); + + Assert.False(result.Success); + Assert.Equal("execution.authorization-denied", result.Code); + Assert.Empty(files.Writes); + Assert.Empty(process.Calls); + } + } + } + + [Fact] + public void ValidGrantForAnotherNodeIsRejectedBeforeMutation() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var otherNodeGrant = SignGrant(authority, signingKey, plan, nodeId: "other-node"); + var localGrant = SignGrant(authority, signingKey, plan); + var requests = new[] + { + CreateExecutionRequest(plan, otherNodeGrant) with + { + Execution = new ProvisioningBaseInstallExecutionAuthorization( + "other-node", plan, otherNodeGrant) + }, + CreateExecutionRequest(plan, localGrant) with + { + Execution = new ProvisioningBaseInstallExecutionAuthorization( + "other-node", plan, localGrant) + } + }; + + foreach (var request in requests) + { + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + var process = new FakeProcessRunner([]); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, + "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(request); + + Assert.False(result.Success); + Assert.Equal("execution.authorization-denied", result.Code); + Assert.Empty(files.Writes); + Assert.Empty(process.Calls); + } + } + } + + [Fact] + public void UnsupportedBaseInstallFieldsCauseZeroMutation() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var baseline = CreateTimezoneOnlyPlan("Europe/Berlin"); + var unsupportedPlans = new[] + { + baseline with { Locale = "en_US.UTF-8" }, + baseline with { AptUpdate = true }, + baseline with { AptUpgrade = true }, + baseline with { Packages = ["curl"] }, + baseline with { SwapMode = "automatic" }, + baseline with { VmSwappiness = 10 }, + baseline with { EnableUnattendedUpgrades = true }, + baseline with { RebootPolicy = "always" } + }; + + foreach (var plan in unsupportedPlans) + { + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + var process = new FakeProcessRunner([]); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest(plan, SignGrant(authority, signingKey, plan))); + + Assert.False(result.Success); + Assert.Equal("system.base-install.unsupported-fields", result.Code); + Assert.Empty(files.Writes); + Assert.Empty(process.Calls); + } + } + } + + [Fact] + public void AgentExecutionRequestUsesTheControlAuthorizedPlanAndGrant() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + using (var parameters = JsonDocument.Parse("{}")) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var grant = SignGrant(authority, signingKey, plan); + var authorization = new ProvisioningBaseInstallExecutionAuthorization("home", plan, grant); + var job = new ProvisioningJob( + new string('d', 32), "home", "system.base-install", 1, + parameters.RootElement.Clone(), ProvisioningJobStates.Running, true, + "Change timezone", "operator", DateTimeOffset.UtcNow.AddMinutes(-1), + DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(30), + DateTimeOffset.UtcNow, null, 3, 40, "execute", null); + + var request = ProvisioningHelperClient.CreateBaseInstallExecutionRequest(job, authorization); + + Assert.Same(authorization, request.Execution); + Assert.Equal(job.Id, request.JobId); + Assert.Equal(job.ActionType, request.ActionType); + Assert.Equal(ProvisioningActionCatalog.SystemBaseInstallModuleHash, request.ModuleHash); + Assert.Equal(JsonValueKind.Object, request.Parameters.ValueKind); + } + } + + [Fact] + public void AgentRejectsInconsistentOrUnverifiedHelperSuccess() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("UTC"); + var authorization = new ProvisioningBaseInstallExecutionAuthorization( + "home", plan, SignGrant(authority, signingKey, plan)); + var unverified = new ProvisioningBaseInstallExecutionResult( + true, "bad", "bad", false, false, false, false, "UTC"); + var contradictory = new ProvisioningHelperResponse( + false, "bad", "bad", null, null, + unverified with { Verified = true }); + + Assert.Throws(() => + ProvisioningHelperClient.ValidateBaseInstallExecutionResponse( + new ProvisioningHelperResponse(true, "bad", "bad", null, null, unverified), + authorization)); + Assert.Throws(() => + ProvisioningHelperClient.ValidateBaseInstallExecutionResponse( + contradictory, authorization)); + } + } + + [Fact] + public void SymlinkedBackupPathIsRejectedBeforeTimezoneMutation() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var files = new FakeFileSystem([]); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + files.SymbolicLinks.Add("/var/lib/ochenstarik-server-monitor-manager/provisioning"); + var process = new FakeProcessRunner( + [], new ProvisioningProcessResult(0, "UTC\n", "")); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest( + plan, SignGrant(authority, signingKey, plan))); + + Assert.False(result.Success); + Assert.Equal("backup.unsafe-path", result.Code); + Assert.Empty(files.Writes); + Assert.DoesNotContain(process.Calls, + call => call.Arguments.Count > 0 && call.Arguments[0] == "set-timezone"); + } + } + + [Fact] + public void VerificationFailureAttemptsAndFactuallyVerifiesRollback() + { + using var authority = CreateAuthority(out var signingKey); + using (signingKey) + { + var plan = CreateTimezoneOnlyPlan("Europe/Berlin"); + var events = new List(); + var files = new FakeFileSystem(events); + files.Files.Add("/usr/share/zoneinfo/Europe/Berlin"); + var process = new FakeProcessRunner(events, + new(0, "UTC\n", ""), + new(0, "", ""), + new(0, "Etc/Unknown\n", ""), + new(0, "", ""), + new(0, "UTC\n", "")); + var executor = new TimezoneProvisioningExecutor( + authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback"); + + var result = executor.Execute(CreateExecutionRequest( + plan, SignGrant(authority, signingKey, plan))); + + Assert.False(result.Success); + Assert.Equal("timezone.verification-failed", result.Code); + Assert.True(result.RollbackAttempted); + Assert.True(result.RollbackSucceeded); + Assert.Equal( + ["set-timezone Europe/Berlin", "set-timezone UTC"], + process.Calls + .Where(call => call.Arguments.Count > 0 && call.Arguments[0] == "set-timezone") + .Select(call => string.Join(' ', call.Arguments))); + } + } + + private static ProvisioningHelperRequest CreateExecutionRequest( + SystemBaseInstallPlan plan, + ProvisioningExecutionGrant grant) + { + using var document = JsonDocument.Parse("{}"); + return new ProvisioningHelperRequest( + "1", new string('d', 32), "system.base-install", 1, + ProvisioningActionCatalog.SystemBaseInstallModuleHash, + document.RootElement.Clone(), + new ProvisioningBaseInstallExecutionAuthorization("home", plan, grant)); + } + + private static SystemBaseInstallPlan CreateTimezoneOnlyPlan(string timezone) + => new(timezone, "unchanged", false, false, [], "unchanged", null, 60, false, "never", []); + + private static X509Certificate2 CreateAuthority(out ECDsa key) + { + key = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var request = new CertificateRequest("CN=SMM Test CA", key, HashAlgorithmName.SHA256); + request.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true)); + return request.CreateSelfSigned( + DateTimeOffset.UtcNow.AddMinutes(-1), DateTimeOffset.UtcNow.AddDays(1)); + } + + private static ProvisioningExecutionGrant SignGrant( + X509Certificate2 authority, + ECDsa key, + SystemBaseInstallPlan plan, + DateTimeOffset? issuedAt = null, + string nodeId = "home") + { + _ = authority; + var now = issuedAt ?? DateTimeOffset.UtcNow; + var grant = new ProvisioningExecutionGrant( + "1", new string('d', 32), nodeId, "system.base-install", 1, + ProvisioningExecutionGrantCodec.ComputePlanSha256(plan), + now.ToUnixTimeSeconds(), now.AddMinutes(2).ToUnixTimeSeconds(), + new string('e', 32), ProvisioningExecutionGrantCodec.SignatureAlgorithm, string.Empty); + var signature = key.SignData( + ProvisioningExecutionGrantCodec.CreateSigningPayload(grant), + HashAlgorithmName.SHA256, + DSASignatureFormat.IeeeP1363FixedFieldConcatenation); + return grant with { Signature = ProvisioningExecutionGrantCodec.EncodeBase64Url(signature) }; + } + + private sealed class FakeFileSystem(List events) : IProvisioningFileSystem + { + public HashSet Files { get; } = new(StringComparer.Ordinal); + public HashSet SymbolicLinks { get; } = new(StringComparer.Ordinal); + public List<(string Path, string Content)> Writes { get; } = []; + + public bool FileExists(string path) => Files.Contains(path); + + public bool IsSymbolicLink(string path) => SymbolicLinks.Contains(path); + + public void CreateOwnerOnlyDirectory(string path) + => events.Add($"mkdir:{path}"); + + public void WriteOwnerOnlyFile(string path, string content) + { + if (Writes.Any(write => string.Equals(write.Path, path, StringComparison.Ordinal))) + { + throw new IOException("Owner-only file already exists."); + } + Writes.Add((path, content)); + events.Add("write-backup"); + } + } + + private sealed class FakeProcessRunner( + List events, + params ProvisioningProcessResult[] results) : IProvisioningProcessRunner + { + private readonly Queue _results = new(results); + public List<(string FileName, IReadOnlyList Arguments)> Calls { get; } = []; + + public ProvisioningProcessResult Run(string fileName, IReadOnlyList arguments) + { + Calls.Add((fileName, arguments.ToArray())); + events.Add($"process:{string.Join(' ', arguments)}"); + return _results.Count == 0 + ? throw new InvalidOperationException("Unexpected process call.") + : _results.Dequeue(); + } + } } diff --git a/tests/bootstrap/test-bootstrap-contract.sh b/tests/bootstrap/test-bootstrap-contract.sh index 135cdee..6beedaa 100755 --- a/tests/bootstrap/test-bootstrap-contract.sh +++ b/tests/bootstrap/test-bootstrap-contract.sh @@ -6,6 +6,11 @@ root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" helper="$root/deploy/ochenstarik-smm-policy-apply" emergency="$root/deploy/ochenstarik-smm-emergency" +provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service" + +grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit" +grep -Fq 'ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback' "$provisioning_helper_unit" +grep -Fq 'install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback"' "$bootstrap" help_output="$(bash "$bootstrap" --help)" version_output="$(bash "$bootstrap" --version)"