Execute confirmed timezone provisioning safely

This commit is contained in:
Ochenstarik 2026-07-30 13:47:07 +07:00
parent 1eae8e1eea
commit 461cb3a3a1
20 changed files with 1138 additions and 47 deletions

View file

@ -541,6 +541,7 @@ install_agent() {
ensure_system_user "$AGENT_USER"
install -d -m 0750 -o root -g "$AGENT_USER" "$ETC_DIR"
install -d -m 0700 -o "$AGENT_USER" -g "$AGENT_USER" "$STATE_DIR/agent"
install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback"
install_tree_atomic "$TEMP_DIR/agent" "$LIB_DIR/agent" "root:root"
install_tree_atomic "$TEMP_DIR/provisioning-helper" "$LIB_DIR/provisioning-helper" "root:root"
install -d -m 0755 "$(dirname "$EMERGENCY_COMMAND")"

View file

@ -7,6 +7,7 @@ Type=simple
User=root
Group=ochenstarik-smm-agent
UMask=0007
EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env
RuntimeDirectory=ochenstarik-server-monitor-manager
RuntimeDirectoryMode=0750
WorkingDirectory=/usr/local/lib/ochenstarik-server-monitor-manager/provisioning-helper
@ -18,6 +19,7 @@ PrivateTmp=true
PrivateDevices=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true

View file

@ -112,11 +112,12 @@
- [x] state machine, confirmations, cancellation, retry и rollback;
- [x] создание, чтение, подтверждение и отмена через Operator API;
- [ ] выполнение, retry, verification и rollback в полной state machine;
- [x] безопасный частичный execution increment для `system.base-install`: timezone-only, factual verification и автоматический rollback;
- [x] обязательные idempotency key, audit reason и job TTL;
- [x] атомарный Agent job channel только для собственного `node_id`;
- [x] начальные строгие JSON schemas v1 для `preflight` и `system.base-install`;
- [ ] versioned JSON schemas для остальных action type;
- [x] restricted root helper через Unix socket (`preflight` и non-mutating plan для `system.base-install`);
- [x] restricted root helper через Unix socket (`preflight`, plan и подтверждённый timezone-only execution для `system.base-install`);
- [x] двухфазный `system.base-install`: сохранённый проверенный plan до Operator confirmation;
- [x] короткоживущий ECDSA execution grant, привязанный к Node, job и SHA-256 подтверждённого plan;
- [x] structured redacted events, bounded Operator history и progress;

View file

@ -165,6 +165,13 @@ internal sealed class AgentClient(AgentOptions options)
return;
}
if (job.ActionType == "system.base-install"
&& job.State == ProvisioningJobStates.Running)
{
await ExecuteBaseInstallAsync(client, job, cancellationToken);
return;
}
if (job.ActionType != "preflight")
{
await ReportProvisioningAsync(
@ -203,6 +210,99 @@ internal sealed class AgentClient(AgentOptions options)
+ $"{result.OperatingSystemVersion} {result.Architecture}.");
}
private async Task ExecuteBaseInstallAsync(
HttpClient client,
ProvisioningJob job,
CancellationToken cancellationToken)
{
ProvisioningBaseInstallExecutionAuthorization authorization;
try
{
authorization = await RequestBaseInstallExecutionAuthorizationAsync(
client, job, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
"authorize", "base-install.authorization-failed",
"Base installation authorization failed before mutation.", cancellationToken);
Console.Error.WriteLine($"Base installation {job.Id} authorization failed: {exception.Message}");
return;
}
ProvisioningBaseInstallExecutionResult result;
try
{
var helper = new ProvisioningHelperClient(options.ProvisioningSocketPath);
result = await helper.ExecuteBaseInstallAsync(job, authorization, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.NeedsReconciliation, job.ProgressPercent,
"reconcile", "base-install.execution-uncertain",
"Base installation outcome is uncertain and requires reconciliation.", cancellationToken);
Console.Error.WriteLine($"Base installation {job.Id} outcome is uncertain: {exception.Message}");
return;
}
if (result.Success)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Verifying, 90,
"verify", "base-install.verifying", "Verifying applied configuration.", cancellationToken);
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Completed, 100,
"completed", "base-install.completed", "Base installation change verified.", cancellationToken);
Console.WriteLine($"Base installation {job.Id} completed and verified.");
return;
}
if (!result.RollbackAttempted)
{
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.Failed, job.ProgressPercent,
"execute", result.Code, "Base installation was rejected before mutation.", cancellationToken);
return;
}
await ReportProvisioningAsync(
client, job, ProvisioningJobStates.RollingBack, 90,
"rollback", "base-install.rollback", "Restoring the previous configuration.", cancellationToken);
await ReportProvisioningAsync(
client, job,
result.RollbackSucceeded ? ProvisioningJobStates.RolledBack : ProvisioningJobStates.RollbackFailed,
100,
result.RollbackSucceeded ? "rolled-back" : "rollback-failed",
result.RollbackSucceeded ? "base-install.rolled-back" : "base-install.rollback-failed",
result.RollbackSucceeded
? "Previous configuration restored and verified."
: "Previous configuration could not be verified after rollback.",
cancellationToken);
}
private static async Task<ProvisioningBaseInstallExecutionAuthorization>
RequestBaseInstallExecutionAuthorizationAsync(
HttpClient client,
ProvisioningJob job,
CancellationToken cancellationToken)
{
var request = new ProvisioningExecutionGrantRequest(
CreateOperationId(job.Id, "execution-grant"));
using var response = await client.PostAsJsonAsync(
$"api/v1/agents/provisioning/jobs/{job.Id}/execution-grant",
request,
SmmJsonContext.Default.ProvisioningExecutionGrantRequest,
cancellationToken);
response.EnsureSuccessStatusCode();
return await response.Content.ReadFromJsonAsync(
SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization,
cancellationToken)
?? throw new InvalidOperationException(
"Control service returned an empty execution authorization.");
}
private static async Task ReportProvisioningAsync(
HttpClient client,
ProvisioningJob job,

View file

@ -39,6 +39,53 @@ public sealed class ProvisioningHelperClient(string socketPath)
return response.BaseInstallPlan;
}
public async Task<ProvisioningBaseInstallExecutionResult> ExecuteBaseInstallAsync(
ProvisioningJob job,
ProvisioningBaseInstallExecutionAuthorization authorization,
CancellationToken cancellationToken)
{
var response = await SendAsync(
CreateBaseInstallExecutionRequest(job, authorization), cancellationToken);
return ValidateBaseInstallExecutionResponse(response, authorization);
}
internal static ProvisioningBaseInstallExecutionResult ValidateBaseInstallExecutionResponse(
ProvisioningHelperResponse response,
ProvisioningBaseInstallExecutionAuthorization authorization)
{
if (response.BaseInstallExecution is null)
{
throw new InvalidOperationException(
$"Provisioning helper returned no execution result: {response.Code}");
}
var result = response.BaseInstallExecution;
if (response.Success != result.Success
|| (result.Success && (!result.Verified
|| result.RollbackAttempted
|| result.RollbackSucceeded
|| !string.Equals(
result.ObservedTimezone,
authorization.Plan.Timezone,
StringComparison.Ordinal)))
|| (result.RollbackSucceeded && !result.RollbackAttempted))
{
throw new InvalidDataException("Provisioning helper returned an inconsistent execution result.");
}
return result;
}
internal static ProvisioningHelperRequest CreateBaseInstallExecutionRequest(
ProvisioningJob job,
ProvisioningBaseInstallExecutionAuthorization authorization)
=> new(
ProvisioningExecutionGrantCodec.ProtocolVersion,
job.Id,
job.ActionType,
job.SchemaVersion,
ProvisioningActionCatalog.SystemBaseInstallModuleHash,
job.Parameters,
authorization);
private async Task<ProvisioningHelperResponse> SendAsync(
ProvisioningHelperRequest request,
CancellationToken cancellationToken)

View file

@ -74,16 +74,25 @@ public sealed class CertificateAuthority : IDisposable
{
if (job.ActionType != "system.base-install"
|| job.SchemaVersion != 1
|| job.State != ProvisioningJobStates.Queued
|| job.State != ProvisioningJobStates.Running
|| !job.ConfirmationRequired
|| job.ConfirmedAt is null
|| lifetime <= TimeSpan.Zero
|| lifetime > ProvisioningExecutionGrantCodec.MaximumLifetime)
{
throw new InvalidOperationException(
"Only a confirmed queued base installation job can receive an execution grant.");
"Only a confirmed running base installation job can receive an execution grant.");
}
var expiresAt = issuedAt.Add(lifetime) < job.ExpiresAt
? issuedAt.Add(lifetime)
: job.ExpiresAt;
var issuedAtUnixSeconds = issuedAt.ToUnixTimeSeconds();
var expiresAtUnixSeconds = expiresAt.ToUnixTimeSeconds();
if (expiresAtUnixSeconds <= issuedAtUnixSeconds)
{
throw new InvalidOperationException("Provisioning job expires before a grant can be issued.");
}
var grant = new ProvisioningExecutionGrant(
ProvisioningExecutionGrantCodec.ProtocolVersion,
job.Id,
@ -91,8 +100,8 @@ public sealed class CertificateAuthority : IDisposable
job.ActionType,
job.SchemaVersion,
ProvisioningExecutionGrantCodec.ComputePlanSha256(plan),
issuedAt.ToUnixTimeSeconds(),
issuedAt.Add(lifetime).ToUnixTimeSeconds(),
issuedAtUnixSeconds,
expiresAtUnixSeconds,
Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(16)),
ProvisioningExecutionGrantCodec.SignatureAlgorithm,
string.Empty);

View file

@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("ServerMonitorManager.Control.Tests")]

View file

@ -133,7 +133,7 @@ public sealed partial class ControlStore
DateTimeOffset.Parse(reader.GetString(2)));
}
public async Task<ProvisioningExecutionGrant?> IssueBaseInstallExecutionGrantAsync(
public async Task<ProvisioningBaseInstallExecutionAuthorization?> IssueBaseInstallExecutionGrantAsync(
string nodeId,
string jobId,
ProvisioningExecutionGrantRequest request,
@ -148,7 +148,7 @@ public sealed partial class ControlStore
request, SmmJsonContext.Default.ProvisioningExecutionGrantRequest);
var cached = await ReadIdempotentAsync(
connection, transaction, operationKey, requestHash,
SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken);
SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization, cancellationToken);
if (cached is not null)
{
await transaction.CommitAsync(cancellationToken);
@ -163,7 +163,7 @@ public sealed partial class ControlStore
}
if (job.ActionType != "system.base-install"
|| job.SchemaVersion != 1
|| job.State != ProvisioningJobStates.Queued
|| job.State != ProvisioningJobStates.Running
|| !job.ConfirmationRequired
|| job.ConfirmedAt is null
|| job.ExpiresAt <= DateTimeOffset.UtcNow)
@ -187,9 +187,10 @@ public sealed partial class ControlStore
planJson, SmmJsonContext.Default.SystemBaseInstallPlan)
?? throw new InvalidDataException("Stored base installation plan is invalid.");
var grant = issueGrant(job, plan);
var authorization = new ProvisioningBaseInstallExecutionAuthorization(nodeId, plan, grant);
await WriteIdempotentAsync(
connection, transaction, operationKey, requestHash, grant,
SmmJsonContext.Default.ProvisioningExecutionGrant, cancellationToken);
connection, transaction, operationKey, requestHash, authorization,
SmmJsonContext.Default.ProvisioningBaseInstallExecutionAuthorization, cancellationToken);
await WriteAuditAsync(
connection, transaction, nodeId, "provisioning.execution-grant.issued", jobId,
JsonSerializer.Serialize(new
@ -201,7 +202,7 @@ public sealed partial class ControlStore
}),
cancellationToken);
await transaction.CommitAsync(cancellationToken);
return grant;
return authorization;
}
}

View file

@ -107,8 +107,13 @@ public sealed partial class ControlStore
command.Transaction = transaction;
command.CommandText = """
UPDATE provisioning_jobs SET
state = CASE WHEN state = $queued THEN $preflight ELSE $rolling_back END,
state = CASE
WHEN state = $queued AND confirmed_at IS NOT NULL THEN $running
WHEN state = $queued THEN $preflight
ELSE $rolling_back
END,
current_step = CASE
WHEN state = $queued AND confirmed_at IS NOT NULL THEN 'execute'
WHEN state = $queued THEN 'preflight'
ELSE 'rollback'
END,
@ -117,22 +122,22 @@ public sealed partial class ControlStore
WHERE id = (
SELECT id FROM provisioning_jobs
WHERE node_id = $node
AND ((state = $queued
AND (confirmation_required = 0 OR confirmed_at IS NULL))
AND (state = $queued
OR (state = $rolling_back AND current_step = 'rollback-queued'))
AND expires_at > $now
ORDER BY CASE WHEN state = $rolling_back THEN 0 ELSE 1 END, created_at, id
LIMIT 1)
AND ((state = $queued
AND (confirmation_required = 0 OR confirmed_at IS NULL))
AND (state = $queued
OR (state = $rolling_back AND current_step = 'rollback-queued'))
RETURNING *;
""";
command.Parameters.AddWithValue("$node", nodeId);
command.Parameters.AddWithValue("$queued", ProvisioningJobStates.Queued);
command.Parameters.AddWithValue("$preflight", ProvisioningJobStates.Preflight);
command.Parameters.AddWithValue("$running", ProvisioningJobStates.Running);
command.Parameters.AddWithValue("$rolling_back", ProvisioningJobStates.RollingBack);
command.Parameters.AddWithValue("$now", now.ToString("O"));
ProvisioningJob? job;
await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
{
@ -656,7 +661,9 @@ internal static class ProvisioningStateMachine
(ProvisioningJobStates.Preflight, ProvisioningJobStates.Running) => true,
(ProvisioningJobStates.Running, ProvisioningJobStates.Running) => true,
(ProvisioningJobStates.Running, ProvisioningJobStates.Verifying) => true,
(ProvisioningJobStates.Running, ProvisioningJobStates.RollingBack) => true,
(ProvisioningJobStates.Verifying, ProvisioningJobStates.Verifying) => true,
(ProvisioningJobStates.Verifying, ProvisioningJobStates.RollingBack) => true,
(ProvisioningJobStates.Verifying, ProvisioningJobStates.Completed) => targetProgress == 100,
(ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollingBack) => true,
(ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack) => targetProgress == 100,

View file

@ -171,14 +171,16 @@ public sealed record ProvisioningHelperRequest(
string ActionType,
int SchemaVersion,
string ModuleHash,
JsonElement Parameters);
JsonElement Parameters,
ProvisioningBaseInstallExecutionAuthorization? Execution = null);
public sealed record ProvisioningHelperResponse(
bool Success,
string Code,
string Message,
ProvisioningPreflightResult? Preflight,
SystemBaseInstallPlan? BaseInstallPlan);
SystemBaseInstallPlan? BaseInstallPlan,
ProvisioningBaseInstallExecutionResult? BaseInstallExecution = null);
[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
public sealed record ProvisioningPreflightResult(
@ -301,6 +303,37 @@ public sealed record ProvisioningExecutionGrant(
[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
public sealed record ProvisioningExecutionGrantRequest(string IdempotencyKey);
[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)]
public sealed record ProvisioningBaseInstallExecutionAuthorization(
string NodeId,
SystemBaseInstallPlan Plan,
ProvisioningExecutionGrant Grant);
public sealed record ProvisioningBaseInstallExecutionResult(
bool Success,
string Code,
string Message,
bool Changed,
bool Verified,
bool RollbackAttempted,
bool RollbackSucceeded,
string? ObservedTimezone);
public sealed record ProvisioningTimezoneRollbackRecord(
int SchemaVersion,
string JobId,
string NodeId,
string PlanSha256,
string PreviousTimezone,
string RequestedTimezone);
public sealed record ProvisioningExecutionConsumptionRecord(
int SchemaVersion,
string JobId,
string NodeId,
string Nonce,
DateTimeOffset ConsumedAt);
public static class PreflightDriftStatuses
{
public const string NotConfigured = "NotConfigured";
@ -430,7 +463,7 @@ public static class SystemBaseInstallSchema
&& parameters.PackageGroupIds.Distinct(StringComparer.Ordinal).Count()
== parameters.PackageGroupIds.Length
&& parameters.PackageGroupIds.All(SystemBaseInstallCatalogDefinition.ContainsGroup)
&& parameters.SwapMode is "disabled" or "automatic" or "explicit"
&& parameters.SwapMode is "unchanged" or "disabled" or "automatic" or "explicit"
&& (parameters.SwapMode == "explicit"
? parameters.SwapSizeMiB is >= 128 and <= 1_048_576
: parameters.SwapSizeMiB is null)

View file

@ -95,7 +95,8 @@ public static class ProvisioningExecutionGrantCodec
{
return false;
}
if (signature.Length != 64)
if (signature.Length != 64
|| !string.Equals(EncodeBase64Url(signature), grant.Signature, StringComparison.Ordinal))
{
return false;
}

View file

@ -47,6 +47,10 @@ namespace ServerMonitorManager.Core;
[JsonSerializable(typeof(ProvisioningBaseInstallPlanRecord))]
[JsonSerializable(typeof(ProvisioningExecutionGrant))]
[JsonSerializable(typeof(ProvisioningExecutionGrantRequest))]
[JsonSerializable(typeof(ProvisioningBaseInstallExecutionAuthorization))]
[JsonSerializable(typeof(ProvisioningBaseInstallExecutionResult))]
[JsonSerializable(typeof(ProvisioningTimezoneRollbackRecord))]
[JsonSerializable(typeof(ProvisioningExecutionConsumptionRecord))]
[JsonSerializable(typeof(ProvisioningJob))]
[JsonSerializable(typeof(ProvisioningJob[]))]
[JsonSerializable(typeof(ProvisioningEvent))]

View file

@ -1,3 +1,4 @@
using System.Security.Cryptography.X509Certificates;
using ServerMonitorManager.Provisioning.Helper;
if (!OperatingSystem.IsLinux())
@ -12,6 +13,16 @@ if (args.Length != 0)
return 2;
}
const string socketPath = "/run/ochenstarik-server-monitor-manager/provisioning.sock";
const string controlAuthorityPath = "/etc/ochenstarik-server-monitor-manager/control-ca.crt";
const string rollbackDirectory =
"/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback";
var localNodeId = Environment.GetEnvironmentVariable("SMM_NodeId");
if (localNodeId is not { Length: >= 1 and <= 63 }
|| !localNodeId.All(character => character is >= 'a' and <= 'z' or >= '0' and <= '9' or '-'))
{
Console.Error.WriteLine("SMM_NodeId must identify the local enrolled Node.");
return 2;
}
using var shutdown = new CancellationTokenSource();
Console.CancelKeyPress += (_, eventArgs) =>
@ -20,5 +31,13 @@ Console.CancelKeyPress += (_, eventArgs) =>
shutdown.Cancel();
};
await new ProvisioningHelperServer(socketPath).RunAsync(shutdown.Token);
using var controlAuthority = X509CertificateLoader.LoadCertificateFromFile(controlAuthorityPath);
var timezoneExecutor = new TimezoneProvisioningExecutor(
controlAuthority,
localNodeId,
new ProvisioningFileSystem(),
new ProvisioningProcessRunner(),
TimeProvider.System,
rollbackDirectory);
await new ProvisioningHelperServer(socketPath, timezoneExecutor).RunAsync(shutdown.Token);
return 0;

View file

@ -7,18 +7,28 @@ using ServerMonitorManager.Core;
namespace ServerMonitorManager.Provisioning.Helper;
public sealed class ProvisioningHelperServer(string socketPath)
public sealed class ProvisioningHelperServer
{
private const int MaximumRequestBytes = 16 * 1024;
private readonly string _socketPath;
private readonly TimezoneProvisioningExecutor? _timezoneExecutor;
public ProvisioningHelperServer(
string socketPath,
TimezoneProvisioningExecutor? timezoneExecutor = null)
{
_socketPath = socketPath;
_timezoneExecutor = timezoneExecutor;
}
[SupportedOSPlatform("linux")]
public async Task RunAsync(CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(socketPath)!);
File.Delete(socketPath);
Directory.CreateDirectory(Path.GetDirectoryName(_socketPath)!);
File.Delete(_socketPath);
using var listener = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified);
listener.Bind(new UnixDomainSocketEndPoint(socketPath));
File.SetUnixFileMode(socketPath,
listener.Bind(new UnixDomainSocketEndPoint(_socketPath));
File.SetUnixFileMode(_socketPath,
UnixFileMode.UserRead | UnixFileMode.UserWrite
| UnixFileMode.GroupRead | UnixFileMode.GroupWrite);
listener.Listen(8);
@ -27,22 +37,39 @@ public sealed class ProvisioningHelperServer(string socketPath)
while (!cancellationToken.IsCancellationRequested)
{
var connection = await listener.AcceptAsync(cancellationToken);
await HandleAsync(connection, cancellationToken);
try
{
await HandleAsync(connection, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception)
{
connection.Dispose();
// A malformed or disconnected local client must not terminate the helper listener.
}
}
}
finally
{
File.Delete(socketPath);
File.Delete(_socketPath);
}
}
public static ProvisioningHelperResponse Execute(ProvisioningHelperRequest request)
=> Execute(request, null);
public static ProvisioningHelperResponse Execute(
ProvisioningHelperRequest request,
TimezoneProvisioningExecutor? timezoneExecutor)
{
if (request.ProtocolVersion != "1")
{
return Failure("protocol.unsupported", "Unsupported helper protocol version.");
}
if (request.JobId.Length != 32 || !request.JobId.All(Uri.IsHexDigit))
if (request.JobId is not { Length: 32 } || !request.JobId.All(Uri.IsHexDigit))
{
return Failure("request.invalid-job", "Invalid provisioning job identifier.");
}
@ -54,11 +81,26 @@ public sealed class ProvisioningHelperServer(string socketPath)
return request.ActionType switch
{
"preflight" => ExecutePreflight(request),
"system.base-install" => CreateBaseInstallPlan(request),
"system.base-install" => request.Execution is null
? CreateBaseInstallPlan(request)
: ExecuteBaseInstall(request, timezoneExecutor),
_ => Failure("action.denied", "The requested action is not allowed.")
};
}
private static ProvisioningHelperResponse ExecuteBaseInstall(
ProvisioningHelperRequest request,
TimezoneProvisioningExecutor? timezoneExecutor)
{
if (timezoneExecutor is null)
{
return Failure("execution.unavailable", "Provisioning execution is unavailable.");
}
var result = timezoneExecutor.Execute(request);
return new ProvisioningHelperResponse(
result.Success, result.Code, result.Message, null, null, result);
}
private static ProvisioningHelperResponse ExecutePreflight(ProvisioningHelperRequest request)
{
if (request.ModuleHash != ProvisioningActionCatalog.PreflightModuleHash
@ -114,7 +156,7 @@ public sealed class ProvisioningHelperServer(string socketPath)
true, "system.base-install.plan-ready", "Base install plan is ready.", null, plan);
}
private static async Task HandleAsync(Socket socket, CancellationToken cancellationToken)
private async Task HandleAsync(Socket socket, CancellationToken cancellationToken)
{
using (socket)
await using (var stream = new NetworkStream(socket, ownsSocket: false))
@ -125,9 +167,13 @@ public sealed class ProvisioningHelperServer(string socketPath)
var payload = await ReadRequestAsync(stream, cancellationToken);
var request = JsonSerializer.Deserialize(payload, SmmJsonContext.Default.ProvisioningHelperRequest)
?? throw new JsonException("Empty request.");
response = Execute(request);
response = Execute(request, _timezoneExecutor);
}
catch (Exception exception) when (exception is JsonException or InvalidDataException)
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception)
{
response = Failure("request.invalid", "Invalid helper request.");
}

View file

@ -0,0 +1,345 @@
using System.Diagnostics;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.Json;
using ServerMonitorManager.Core;
namespace ServerMonitorManager.Provisioning.Helper;
public interface IProvisioningFileSystem
{
bool FileExists(string path);
bool IsSymbolicLink(string path);
void CreateOwnerOnlyDirectory(string path);
void WriteOwnerOnlyFile(string path, string content);
}
public interface IProvisioningProcessRunner
{
ProvisioningProcessResult Run(string fileName, IReadOnlyList<string> arguments);
}
public sealed record ProvisioningProcessResult(int ExitCode, string StandardOutput, string StandardError);
public sealed class TimezoneProvisioningExecutor(
X509Certificate2 controlAuthority,
string localNodeId,
IProvisioningFileSystem fileSystem,
IProvisioningProcessRunner processRunner,
TimeProvider timeProvider,
string rollbackDirectory)
{
private const string TimedatectlPath = "/usr/bin/timedatectl";
private const string ZoneinfoRoot = "/usr/share/zoneinfo";
public ProvisioningBaseInstallExecutionResult Execute(ProvisioningHelperRequest request)
{
var authorization = request.Execution;
if (authorization is null
|| request.ProtocolVersion != ProvisioningExecutionGrantCodec.ProtocolVersion
|| request.JobId is not { Length: 32 }
|| !request.JobId.All(Uri.IsHexDigit)
|| request.ActionType != "system.base-install"
|| request.SchemaVersion != 1
|| request.ModuleHash != ProvisioningActionCatalog.SystemBaseInstallModuleHash
|| !string.Equals(authorization.NodeId, localNodeId, StringComparison.Ordinal)
|| !ProvisioningExecutionGrantCodec.Verify(
authorization.Grant,
controlAuthority,
request.JobId,
localNodeId,
authorization.Plan,
timeProvider.GetUtcNow()))
{
return Failure("execution.authorization-denied", "Execution authorization was rejected.");
}
var plan = authorization.Plan;
if (!IsTimezoneOnly(plan))
{
return Failure(
"system.base-install.unsupported-fields",
"Only timezone changes are supported by this execution increment.");
}
if (!IsSafeTimezone(plan.Timezone)
|| !fileSystem.FileExists($"{ZoneinfoRoot}/{plan.Timezone}"))
{
return Failure("timezone.invalid", "The requested timezone is unavailable.");
}
if (!rollbackDirectory.StartsWith("/", StringComparison.Ordinal)
|| HasSymlinkedManagedPath(rollbackDirectory))
{
return Failure("backup.unsafe-path", "The rollback path is unsafe.");
}
try
{
fileSystem.CreateOwnerOnlyDirectory(rollbackDirectory);
var nonceHash = Convert.ToHexStringLower(
SHA256.HashData(Encoding.UTF8.GetBytes(authorization.Grant.Nonce)));
var consumptionPath = $"{rollbackDirectory}/grant-{nonceHash}.consumed.json";
if (fileSystem.IsSymbolicLink(consumptionPath))
{
return Failure("backup.unsafe-path", "The grant consumption path is unsafe.");
}
fileSystem.WriteOwnerOnlyFile(
consumptionPath,
JsonSerializer.Serialize(
new ProvisioningExecutionConsumptionRecord(
1,
request.JobId,
localNodeId,
authorization.Grant.Nonce,
timeProvider.GetUtcNow()),
SmmJsonContext.Default.ProvisioningExecutionConsumptionRecord));
}
catch
{
return Failure(
"execution.grant-consumed",
"Execution grant was already consumed or could not be recorded.");
}
ProvisioningProcessResult observedBefore;
try
{
observedBefore = QueryTimezone();
}
catch
{
return Failure("timezone.inspect-failed", "The current timezone could not be inspected.");
}
var previousTimezone = NormalizeTimezone(observedBefore);
if (previousTimezone is null || !IsSafeTimezone(previousTimezone))
{
return Failure("timezone.inspect-failed", "The current timezone could not be inspected.");
}
if (string.Equals(previousTimezone, plan.Timezone, StringComparison.Ordinal))
{
return new ProvisioningBaseInstallExecutionResult(
true, "timezone.already-configured", "Timezone is already configured.",
false, true, false, false, previousTimezone);
}
try
{
var rollbackPath = $"{rollbackDirectory}/{request.JobId}.json";
if (fileSystem.IsSymbolicLink(rollbackPath))
{
return Failure("backup.unsafe-path", "The rollback path is unsafe.");
}
fileSystem.WriteOwnerOnlyFile(
rollbackPath,
CreateRollbackRecord(request, authorization, previousTimezone));
}
catch
{
return Failure("backup.create-failed", "The rollback record could not be created.");
}
var failureCode = "timezone.mutation-failed";
try
{
var mutation = SetTimezone(plan.Timezone);
if (mutation.ExitCode == 0)
{
failureCode = "timezone.verification-failed";
var observedAfter = NormalizeTimezone(QueryTimezone());
if (string.Equals(observedAfter, plan.Timezone, StringComparison.Ordinal))
{
return new ProvisioningBaseInstallExecutionResult(
true, "timezone.changed", "Timezone was changed and verified.",
true, true, false, false, observedAfter);
}
}
}
catch
{
// A failed process or verification can still have partially changed the host.
}
var rollbackSucceeded = false;
string? observedRollback = null;
try
{
var rollback = SetTimezone(previousTimezone);
if (rollback.ExitCode == 0)
{
observedRollback = NormalizeTimezone(QueryTimezone());
rollbackSucceeded = string.Equals(
observedRollback, previousTimezone, StringComparison.Ordinal);
}
}
catch
{
rollbackSucceeded = false;
}
return new ProvisioningBaseInstallExecutionResult(
false,
failureCode,
failureCode == "timezone.verification-failed"
? "Timezone verification failed; rollback was attempted."
: "Timezone mutation failed; rollback was attempted.",
false,
false,
true,
rollbackSucceeded,
observedRollback);
}
private ProvisioningProcessResult QueryTimezone()
=> processRunner.Run(
TimedatectlPath,
["show", "--property=Timezone", "--value"]);
private ProvisioningProcessResult SetTimezone(string timezone)
=> processRunner.Run(TimedatectlPath, ["set-timezone", timezone]);
private bool HasSymlinkedManagedPath(string path)
{
var current = string.Empty;
foreach (var component in path.Split('/', StringSplitOptions.RemoveEmptyEntries))
{
current += $"/{component}";
if (fileSystem.IsSymbolicLink(current))
{
return true;
}
}
return false;
}
private static string CreateRollbackRecord(
ProvisioningHelperRequest request,
ProvisioningBaseInstallExecutionAuthorization authorization,
string previousTimezone)
=> JsonSerializer.Serialize(
new ProvisioningTimezoneRollbackRecord(
1,
request.JobId,
authorization.NodeId,
authorization.Grant.PlanSha256,
previousTimezone,
authorization.Plan.Timezone),
SmmJsonContext.Default.ProvisioningTimezoneRollbackRecord)
+ "\n";
private static bool IsTimezoneOnly(SystemBaseInstallPlan plan)
=> string.Equals(plan.Locale, "unchanged", StringComparison.Ordinal)
&& !plan.AptUpdate
&& !plan.AptUpgrade
&& plan.Packages is { Length: 0 }
&& string.Equals(plan.SwapMode, "unchanged", StringComparison.Ordinal)
&& plan.SwapSizeMiB is null
&& plan.VmSwappiness == 60
&& !plan.EnableUnattendedUpgrades
&& string.Equals(plan.RebootPolicy, "never", StringComparison.Ordinal);
private static bool IsSafeTimezone(string? value)
=> value is { Length: >= 1 and <= 64 }
&& value[0] is not '/' and not '.'
&& !value.Contains("..", StringComparison.Ordinal)
&& value.All(character => char.IsAsciiLetterOrDigit(character)
|| character is '/' or '_' or '-' or '+');
private static string? NormalizeTimezone(ProvisioningProcessResult result)
{
if (result.ExitCode != 0)
{
return null;
}
var value = result.StandardOutput.Trim();
return value.Length == 0 || value.Contains('\n') || value.Contains('\r')
? null
: value;
}
private static ProvisioningBaseInstallExecutionResult Failure(string code, string message)
=> new(false, code, message, false, false, false, false, null);
}
public sealed class ProvisioningFileSystem : IProvisioningFileSystem
{
public bool FileExists(string path) => File.Exists(path);
public bool IsSymbolicLink(string path)
{
var info = File.Exists(path)
? (FileSystemInfo)new FileInfo(path)
: Directory.Exists(path)
? new DirectoryInfo(path)
: new FileInfo(path);
return info.Exists && info.LinkTarget is not null;
}
public void CreateOwnerOnlyDirectory(string path)
{
Directory.CreateDirectory(path);
if (OperatingSystem.IsLinux())
{
File.SetUnixFileMode(path,
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute);
}
}
public void WriteOwnerOnlyFile(string path, string content)
{
var options = new FileStreamOptions
{
Mode = FileMode.CreateNew,
Access = FileAccess.Write,
Share = FileShare.None,
BufferSize = 4096,
Options = FileOptions.WriteThrough
};
if (OperatingSystem.IsLinux())
{
options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
}
using var stream = new FileStream(path, options);
using var writer = new StreamWriter(stream, new UTF8Encoding(false));
writer.Write(content);
writer.Flush();
stream.Flush(flushToDisk: true);
}
}
public sealed class ProvisioningProcessRunner : IProvisioningProcessRunner
{
public ProvisioningProcessResult Run(string fileName, IReadOnlyList<string> arguments)
{
if (!string.Equals(fileName, "/usr/bin/timedatectl", StringComparison.Ordinal))
{
throw new InvalidOperationException("Executable is outside the provisioning allowlist.");
}
var startInfo = new ProcessStartInfo
{
FileName = fileName,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};
foreach (var argument in arguments)
{
startInfo.ArgumentList.Add(argument);
}
using var process = Process.Start(startInfo)
?? throw new InvalidOperationException("Provisioning process could not be started.");
var standardOutput = process.StandardOutput.ReadToEndAsync();
var standardError = process.StandardError.ReadToEndAsync();
if (!process.WaitForExit(30_000))
{
process.Kill(entireProcessTree: true);
process.WaitForExit();
throw new TimeoutException("Provisioning process timed out.");
}
Task.WaitAll(standardOutput, standardError);
return new ProvisioningProcessResult(
process.ExitCode,
standardOutput.GetAwaiter().GetResult(),
standardError.GetAwaiter().GetResult());
}
}

View file

@ -79,9 +79,9 @@ public sealed class CertificateAuthorityTests : IDisposable
SmmJsonContext.Default.SystemBaseInstallParameters);
var job = new ProvisioningJob(
Guid.NewGuid().ToString("N"), "home", "system.base-install", 1, parameters,
ProvisioningJobStates.Queued, true, "test", "operator",
ProvisioningJobStates.Running, true, "test", "operator",
now.AddMinutes(-1), now, now.AddMinutes(30), now, null, 4, 25,
"confirmed-queued", null);
"execute", null);
var plan = new SystemBaseInstallPlan(
"UTC", "en_US.UTF-8", true, false,
SystemBaseInstallCatalogDefinition.ExpandGroups(["core"]),
@ -99,6 +99,15 @@ public sealed class CertificateAuthorityTests : IDisposable
plan with { Packages = [.. plan.Packages, "untrusted-package"] }, now));
Assert.False(ProvisioningExecutionGrantCodec.Verify(
grant, authority.PublicCertificate, job.Id, job.NodeId, plan, now.AddMinutes(3)));
var expiringJob = job with { ExpiresAt = now.AddSeconds(15) };
var clamped = authority.SignProvisioningExecutionGrant(
expiringJob, plan, now, TimeSpan.FromMinutes(2));
Assert.Equal(expiringJob.ExpiresAt.ToUnixTimeSeconds(), clamped.ExpiresAtUnixSeconds);
Assert.Throws<InvalidOperationException>(() => authority.SignProvisioningExecutionGrant(
job with { ExpiresAt = now }, plan, now, TimeSpan.FromMinutes(2)));
Assert.Throws<InvalidOperationException>(() => authority.SignProvisioningExecutionGrant(
job with { ExpiresAt = now.AddMilliseconds(100) },
plan, now, TimeSpan.FromMinutes(2)));
Assert.Throws<InvalidOperationException>(() => authority.SignProvisioningExecutionGrant(
job with { ConfirmedAt = null }, plan, now, TimeSpan.FromMinutes(2)));
}

View file

@ -181,29 +181,36 @@ public sealed class ControlApiTests : IAsyncDisposable
ServerMonitorManager.Core.ProvisioningJob>(cancellationToken);
Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Queued, confirmed!.State);
Assert.Equal("confirmed-queued", confirmed.CurrentStep);
var execution = await agent.GetFromJsonAsync<ServerMonitorManager.Core.ProvisioningJob>(
"/api/v1/agents/provisioning/jobs/next", cancellationToken);
Assert.Equal(ServerMonitorManager.Core.ProvisioningJobStates.Running, execution!.State);
var grantRequest = new { idempotencyKey = Guid.NewGuid().ToString() };
var grantResponse = await agent.PostAsJsonAsync(
$"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant",
grantRequest,
cancellationToken);
Assert.Equal(HttpStatusCode.OK, grantResponse.StatusCode);
var grant = await grantResponse.Content.ReadFromJsonAsync<
ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken);
var authorization = await grantResponse.Content.ReadFromJsonAsync<
ServerMonitorManager.Core.ProvisioningBaseInstallExecutionAuthorization>(cancellationToken);
var grantReplayResponse = await agent.PostAsJsonAsync(
$"/api/v1/agents/provisioning/jobs/{job.Id}/execution-grant",
grantRequest,
cancellationToken);
var grantReplay = await grantReplayResponse.Content.ReadFromJsonAsync<
ServerMonitorManager.Core.ProvisioningExecutionGrant>(cancellationToken);
Assert.Equal(grant!.Signature, grantReplay!.Signature);
var authorizationReplay = await grantReplayResponse.Content.ReadFromJsonAsync<
ServerMonitorManager.Core.ProvisioningBaseInstallExecutionAuthorization>(cancellationToken);
Assert.Equal(authorization!.Grant.Signature, authorizationReplay!.Grant.Signature);
Assert.Equal(
ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.ComputePlanSha256(storedPlan.Plan),
ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.ComputePlanSha256(authorization.Plan));
Assert.Equal("home", authorization.NodeId);
var authority = _factory.Services.GetRequiredService<
ServerMonitorManager.Control.CertificateAuthority>();
Assert.True(ServerMonitorManager.Core.ProvisioningExecutionGrantCodec.Verify(
grant,
authorization.Grant,
authority.PublicCertificate,
job.Id,
"home",
storedPlan.Plan,
authorization.Plan,
DateTimeOffset.UtcNow));
Assert.Equal(HttpStatusCode.NoContent, (await agent.GetAsync(
"/api/v1/agents/provisioning/jobs/next", cancellationToken)).StatusCode);

View file

@ -11,6 +11,19 @@ public sealed class ControlStoreTests : IAsyncDisposable
{
private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-tests-{Guid.NewGuid():N}");
[Fact]
public void VerifiedHelperRollbackCanBeReportedToControl()
{
Assert.True(ProvisioningStateMachine.CanReport(
ProvisioningJobStates.Running, ProvisioningJobStates.RollingBack, 40, 90));
Assert.True(ProvisioningStateMachine.CanReport(
ProvisioningJobStates.Verifying, ProvisioningJobStates.RollingBack, 90, 90));
Assert.True(ProvisioningStateMachine.CanReport(
ProvisioningJobStates.RollingBack, ProvisioningJobStates.RolledBack, 90, 100));
Assert.True(ProvisioningStateMachine.CanReport(
ProvisioningJobStates.RollingBack, ProvisioningJobStates.RollbackFailed, 90, 100));
}
[Fact]
public async Task VersionOneDatabaseMigratesToProvisioningSchema()
{
@ -99,19 +112,38 @@ public sealed class ControlStoreTests : IAsyncDisposable
Assert.Equal(ProvisioningJobStates.Queued, confirmed.State);
Assert.Equal("confirmed-queued", confirmed.CurrentStep);
Assert.NotNull(confirmed.ConfirmedAt);
var execution = await store.ClaimNextProvisioningJobAsync("home", cancellationToken);
Assert.Equal(ProvisioningJobStates.Running, execution!.State);
Assert.Equal("execute", execution.CurrentStep);
Assert.Null(await store.ClaimNextProvisioningJobAsync("home", cancellationToken));
var cancelled = await store.CancelProvisioningJobAsync(
var rollingBack = await store.ReportProvisioningProgressAsync(
"home",
created.Id,
new ProvisioningJobCommandRequest("Test completed", Guid.NewGuid().ToString()),
"operator",
new ProvisioningJobProgressRequest(
ProvisioningJobStates.RollingBack, 90, "rollback",
"execution.rollback", "Restoring previous state.", Guid.NewGuid().ToString()),
cancellationToken);
Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State);
Assert.NotNull(cancelled.CancelledAt);
Assert.Equal(ProvisioningJobStates.RollingBack, rollingBack!.State);
var rolledBack = await store.ReportProvisioningProgressAsync(
"home",
created.Id,
new ProvisioningJobProgressRequest(
ProvisioningJobStates.RolledBack, 100, "rolled-back",
"execution.rolled-back", "Previous state restored.", Guid.NewGuid().ToString()),
cancellationToken);
Assert.Equal(ProvisioningJobStates.RolledBack, rolledBack!.State);
var next = await store.CreateProvisioningJobAsync(
"home", request with { IdempotencyKey = Guid.NewGuid().ToString() }, "operator", cancellationToken);
Assert.Equal(ProvisioningJobStates.Queued, next.State);
var cancelled = await store.CancelProvisioningJobAsync(
next.Id,
new ProvisioningJobCommandRequest("Cancel before execution", Guid.NewGuid().ToString()),
"operator",
cancellationToken);
Assert.Equal(ProvisioningJobStates.Cancelled, cancelled!.State);
Assert.NotNull(cancelled.CancelledAt);
}
[Fact]

View file

@ -1,4 +1,7 @@
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text.Json;
using ServerMonitorManager.Agent;
using ServerMonitorManager.Core;
using ServerMonitorManager.Provisioning.Helper;
using Xunit;
@ -54,6 +57,18 @@ public sealed class ProvisioningHelperTests
JsonSerializer.Deserialize(json, SmmJsonContext.Default.ProvisioningHelperRequest));
}
[Fact]
public void HelperRejectsMissingJobIdWithoutThrowing()
{
using var document = JsonDocument.Parse("{}");
var response = ProvisioningHelperServer.Execute(new ProvisioningHelperRequest(
"1", null!, "preflight", 1,
ProvisioningActionCatalog.PreflightModuleHash, document.RootElement.Clone()));
Assert.False(response.Success);
Assert.Equal("request.invalid-job", response.Code);
}
[Fact]
public void BaseInstallSchemaRejectsCommandText()
{
@ -88,4 +103,408 @@ public sealed class ProvisioningHelperTests
response.BaseInstallPlan!.Packages);
Assert.Equal("never", response.BaseInstallPlan.RebootPolicy);
}
[Fact]
public void ConfirmedTimezoneOnlyPlanUsesAllowlistedBinaryCreatesBackupAndVerifies()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var grant = SignGrant(authority, signingKey, plan);
var events = new List<string>();
var files = new FakeFileSystem(events);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
var process = new FakeProcessRunner(events,
new(0, "UTC\n", ""),
new(0, "", ""),
new(0, "Europe/Berlin\n", ""));
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(plan, grant));
Assert.True(result.Success);
Assert.True(result.Changed);
Assert.True(result.Verified);
Assert.False(result.RollbackAttempted);
Assert.Equal("Europe/Berlin", result.ObservedTimezone);
Assert.Equal(2, files.Writes.Count);
Assert.True(events.IndexOf("write-backup") < events.IndexOf("process:set-timezone Europe/Berlin"));
Assert.All(process.Calls, call => Assert.Equal("/usr/bin/timedatectl", call.FileName));
Assert.Equal(
[
"show --property=Timezone --value",
"set-timezone Europe/Berlin",
"show --property=Timezone --value"
],
process.Calls.Select(call => string.Join(' ', call.Arguments)));
}
}
[Fact]
public void ValidNoOpTimezoneCompletesOnlyAfterFactualVerification()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("UTC");
var process = new FakeProcessRunner(
[], new ProvisioningProcessResult(0, "UTC\n", ""));
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/UTC");
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(plan, SignGrant(authority, signingKey, plan)));
Assert.True(result.Success);
Assert.False(result.Changed);
Assert.True(result.Verified);
Assert.Single(files.Writes);
Assert.Single(process.Calls);
}
}
[Fact]
public void ConsumedGrantCannotBeReplayedEvenAfterVerifiedNoOp()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("UTC");
var grant = SignGrant(authority, signingKey, plan);
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/UTC");
var firstProcess = new FakeProcessRunner(
[], new ProvisioningProcessResult(0, "UTC\n", ""));
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, firstProcess, TimeProvider.System,
"/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var first = executor.Execute(CreateExecutionRequest(plan, grant));
var secondProcess = new FakeProcessRunner([]);
var replayExecutor = new TimezoneProvisioningExecutor(
authority, "home", files, secondProcess, TimeProvider.System,
"/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var replay = replayExecutor.Execute(CreateExecutionRequest(plan, grant));
Assert.True(first.Success);
Assert.False(replay.Success);
Assert.Equal("execution.grant-consumed", replay.Code);
Assert.Empty(secondProcess.Calls);
}
}
[Fact]
public void ForgedMismatchedAndExpiredGrantsCauseZeroMutation()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var valid = SignGrant(authority, signingKey, plan);
var invalid = new[]
{
valid with { Signature = valid.Signature[..^1] + (valid.Signature[^1] == 'A' ? "B" : "A") },
valid with { JobId = new string('f', 32) },
valid with { NodeId = "other-node" },
valid with { PlanSha256 = new string('0', 64) },
SignGrant(authority, signingKey, plan, DateTimeOffset.UtcNow.AddMinutes(-10))
};
foreach (var grant in invalid)
{
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
var process = new FakeProcessRunner([]);
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(plan, grant));
Assert.False(result.Success);
Assert.Equal("execution.authorization-denied", result.Code);
Assert.Empty(files.Writes);
Assert.Empty(process.Calls);
}
}
}
[Fact]
public void ValidGrantForAnotherNodeIsRejectedBeforeMutation()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var otherNodeGrant = SignGrant(authority, signingKey, plan, nodeId: "other-node");
var localGrant = SignGrant(authority, signingKey, plan);
var requests = new[]
{
CreateExecutionRequest(plan, otherNodeGrant) with
{
Execution = new ProvisioningBaseInstallExecutionAuthorization(
"other-node", plan, otherNodeGrant)
},
CreateExecutionRequest(plan, localGrant) with
{
Execution = new ProvisioningBaseInstallExecutionAuthorization(
"other-node", plan, localGrant)
}
};
foreach (var request in requests)
{
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
var process = new FakeProcessRunner([]);
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System,
"/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(request);
Assert.False(result.Success);
Assert.Equal("execution.authorization-denied", result.Code);
Assert.Empty(files.Writes);
Assert.Empty(process.Calls);
}
}
}
[Fact]
public void UnsupportedBaseInstallFieldsCauseZeroMutation()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var baseline = CreateTimezoneOnlyPlan("Europe/Berlin");
var unsupportedPlans = new[]
{
baseline with { Locale = "en_US.UTF-8" },
baseline with { AptUpdate = true },
baseline with { AptUpgrade = true },
baseline with { Packages = ["curl"] },
baseline with { SwapMode = "automatic" },
baseline with { VmSwappiness = 10 },
baseline with { EnableUnattendedUpgrades = true },
baseline with { RebootPolicy = "always" }
};
foreach (var plan in unsupportedPlans)
{
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
var process = new FakeProcessRunner([]);
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(plan, SignGrant(authority, signingKey, plan)));
Assert.False(result.Success);
Assert.Equal("system.base-install.unsupported-fields", result.Code);
Assert.Empty(files.Writes);
Assert.Empty(process.Calls);
}
}
}
[Fact]
public void AgentExecutionRequestUsesTheControlAuthorizedPlanAndGrant()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
using (var parameters = JsonDocument.Parse("{}"))
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var grant = SignGrant(authority, signingKey, plan);
var authorization = new ProvisioningBaseInstallExecutionAuthorization("home", plan, grant);
var job = new ProvisioningJob(
new string('d', 32), "home", "system.base-install", 1,
parameters.RootElement.Clone(), ProvisioningJobStates.Running, true,
"Change timezone", "operator", DateTimeOffset.UtcNow.AddMinutes(-1),
DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(30),
DateTimeOffset.UtcNow, null, 3, 40, "execute", null);
var request = ProvisioningHelperClient.CreateBaseInstallExecutionRequest(job, authorization);
Assert.Same(authorization, request.Execution);
Assert.Equal(job.Id, request.JobId);
Assert.Equal(job.ActionType, request.ActionType);
Assert.Equal(ProvisioningActionCatalog.SystemBaseInstallModuleHash, request.ModuleHash);
Assert.Equal(JsonValueKind.Object, request.Parameters.ValueKind);
}
}
[Fact]
public void AgentRejectsInconsistentOrUnverifiedHelperSuccess()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("UTC");
var authorization = new ProvisioningBaseInstallExecutionAuthorization(
"home", plan, SignGrant(authority, signingKey, plan));
var unverified = new ProvisioningBaseInstallExecutionResult(
true, "bad", "bad", false, false, false, false, "UTC");
var contradictory = new ProvisioningHelperResponse(
false, "bad", "bad", null, null,
unverified with { Verified = true });
Assert.Throws<InvalidDataException>(() =>
ProvisioningHelperClient.ValidateBaseInstallExecutionResponse(
new ProvisioningHelperResponse(true, "bad", "bad", null, null, unverified),
authorization));
Assert.Throws<InvalidDataException>(() =>
ProvisioningHelperClient.ValidateBaseInstallExecutionResponse(
contradictory, authorization));
}
}
[Fact]
public void SymlinkedBackupPathIsRejectedBeforeTimezoneMutation()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var files = new FakeFileSystem([]);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
files.SymbolicLinks.Add("/var/lib/ochenstarik-server-monitor-manager/provisioning");
var process = new FakeProcessRunner(
[], new ProvisioningProcessResult(0, "UTC\n", ""));
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(
plan, SignGrant(authority, signingKey, plan)));
Assert.False(result.Success);
Assert.Equal("backup.unsafe-path", result.Code);
Assert.Empty(files.Writes);
Assert.DoesNotContain(process.Calls,
call => call.Arguments.Count > 0 && call.Arguments[0] == "set-timezone");
}
}
[Fact]
public void VerificationFailureAttemptsAndFactuallyVerifiesRollback()
{
using var authority = CreateAuthority(out var signingKey);
using (signingKey)
{
var plan = CreateTimezoneOnlyPlan("Europe/Berlin");
var events = new List<string>();
var files = new FakeFileSystem(events);
files.Files.Add("/usr/share/zoneinfo/Europe/Berlin");
var process = new FakeProcessRunner(events,
new(0, "UTC\n", ""),
new(0, "", ""),
new(0, "Etc/Unknown\n", ""),
new(0, "", ""),
new(0, "UTC\n", ""));
var executor = new TimezoneProvisioningExecutor(
authority, "home", files, process, TimeProvider.System, "/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback");
var result = executor.Execute(CreateExecutionRequest(
plan, SignGrant(authority, signingKey, plan)));
Assert.False(result.Success);
Assert.Equal("timezone.verification-failed", result.Code);
Assert.True(result.RollbackAttempted);
Assert.True(result.RollbackSucceeded);
Assert.Equal(
["set-timezone Europe/Berlin", "set-timezone UTC"],
process.Calls
.Where(call => call.Arguments.Count > 0 && call.Arguments[0] == "set-timezone")
.Select(call => string.Join(' ', call.Arguments)));
}
}
private static ProvisioningHelperRequest CreateExecutionRequest(
SystemBaseInstallPlan plan,
ProvisioningExecutionGrant grant)
{
using var document = JsonDocument.Parse("{}");
return new ProvisioningHelperRequest(
"1", new string('d', 32), "system.base-install", 1,
ProvisioningActionCatalog.SystemBaseInstallModuleHash,
document.RootElement.Clone(),
new ProvisioningBaseInstallExecutionAuthorization("home", plan, grant));
}
private static SystemBaseInstallPlan CreateTimezoneOnlyPlan(string timezone)
=> new(timezone, "unchanged", false, false, [], "unchanged", null, 60, false, "never", []);
private static X509Certificate2 CreateAuthority(out ECDsa key)
{
key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var request = new CertificateRequest("CN=SMM Test CA", key, HashAlgorithmName.SHA256);
request.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true));
return request.CreateSelfSigned(
DateTimeOffset.UtcNow.AddMinutes(-1), DateTimeOffset.UtcNow.AddDays(1));
}
private static ProvisioningExecutionGrant SignGrant(
X509Certificate2 authority,
ECDsa key,
SystemBaseInstallPlan plan,
DateTimeOffset? issuedAt = null,
string nodeId = "home")
{
_ = authority;
var now = issuedAt ?? DateTimeOffset.UtcNow;
var grant = new ProvisioningExecutionGrant(
"1", new string('d', 32), nodeId, "system.base-install", 1,
ProvisioningExecutionGrantCodec.ComputePlanSha256(plan),
now.ToUnixTimeSeconds(), now.AddMinutes(2).ToUnixTimeSeconds(),
new string('e', 32), ProvisioningExecutionGrantCodec.SignatureAlgorithm, string.Empty);
var signature = key.SignData(
ProvisioningExecutionGrantCodec.CreateSigningPayload(grant),
HashAlgorithmName.SHA256,
DSASignatureFormat.IeeeP1363FixedFieldConcatenation);
return grant with { Signature = ProvisioningExecutionGrantCodec.EncodeBase64Url(signature) };
}
private sealed class FakeFileSystem(List<string> events) : IProvisioningFileSystem
{
public HashSet<string> Files { get; } = new(StringComparer.Ordinal);
public HashSet<string> SymbolicLinks { get; } = new(StringComparer.Ordinal);
public List<(string Path, string Content)> Writes { get; } = [];
public bool FileExists(string path) => Files.Contains(path);
public bool IsSymbolicLink(string path) => SymbolicLinks.Contains(path);
public void CreateOwnerOnlyDirectory(string path)
=> events.Add($"mkdir:{path}");
public void WriteOwnerOnlyFile(string path, string content)
{
if (Writes.Any(write => string.Equals(write.Path, path, StringComparison.Ordinal)))
{
throw new IOException("Owner-only file already exists.");
}
Writes.Add((path, content));
events.Add("write-backup");
}
}
private sealed class FakeProcessRunner(
List<string> events,
params ProvisioningProcessResult[] results) : IProvisioningProcessRunner
{
private readonly Queue<ProvisioningProcessResult> _results = new(results);
public List<(string FileName, IReadOnlyList<string> Arguments)> Calls { get; } = [];
public ProvisioningProcessResult Run(string fileName, IReadOnlyList<string> arguments)
{
Calls.Add((fileName, arguments.ToArray()));
events.Add($"process:{string.Join(' ', arguments)}");
return _results.Count == 0
? throw new InvalidOperationException("Unexpected process call.")
: _results.Dequeue();
}
}
}

View file

@ -6,6 +6,11 @@ root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
helper="$root/deploy/ochenstarik-smm-policy-apply"
emergency="$root/deploy/ochenstarik-smm-emergency"
provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service"
grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit"
grep -Fq 'ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/provisioning/rollback' "$provisioning_helper_unit"
grep -Fq 'install -d -m 0700 -o root -g root "$STATE_DIR/provisioning/rollback"' "$bootstrap"
help_output="$(bash "$bootstrap" --help)"
version_output="$(bash "$bootstrap" --version)"