feat(control): add background link reconciliation
This commit is contained in:
Ochenstarik
parent
89ef2fd9d3
commit
c757920791
21 changed files with 1116 additions and 175 deletions
|
|
@ -494,6 +494,7 @@ Control__CertificateAuthorityPath=$ETC_DIR/control-ca.pfx
|
|||
Control__BackupDirectory=$STATE_DIR/backups
|
||||
Control__HubHelperPath=$POLICY_HELPER
|
||||
Control__PrivilegeEscalationPath=/usr/bin/sudo
|
||||
Control__LinkReconciliationSeconds=300
|
||||
EOF
|
||||
printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url"
|
||||
chown root:"$CONTROL_USER" "$ETC_DIR/control.env"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
IFS=$'\n\t'
|
||||
umask 077
|
||||
|
||||
readonly PROGRAM="ochenstarik-smm-emergency"
|
||||
readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager"
|
||||
readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager"
|
||||
readonly MARKER="$STATE_DIR/mesh/emergency-disabled"
|
||||
readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"
|
||||
readonly RECONCILE_LOCK="$STATE_DIR/mesh/reconcile-requested.lock"
|
||||
readonly CONTROL_UNIT="ochenstarik-smm-control.service"
|
||||
readonly AGENT_UNIT="ochenstarik-smm-agent.service"
|
||||
readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service"
|
||||
|
|
@ -71,6 +74,25 @@ delete_project_firewall() {
|
|||
fi
|
||||
}
|
||||
|
||||
request_reconciliation() {
|
||||
local marker_directory temporary_marker generation
|
||||
marker_directory="$(dirname "$RECONCILE_MARKER")"
|
||||
install -d -o root -g root -m 0700 "$marker_directory"
|
||||
exec 9>"$RECONCILE_LOCK"
|
||||
chown root:root "$RECONCILE_LOCK"
|
||||
chmod 0600 "$RECONCILE_LOCK"
|
||||
/usr/bin/flock -x 9
|
||||
generation="$(</proc/sys/kernel/random/uuid)"
|
||||
temporary_marker="$(mktemp "$marker_directory/.reconcile-requested.XXXXXXXX")"
|
||||
if ! printf '%s\n' "$generation" >"$temporary_marker" \
|
||||
|| ! chown root:root "$temporary_marker" \
|
||||
|| ! chmod 0600 "$temporary_marker" \
|
||||
|| ! mv -f -- "$temporary_marker" "$RECONCILE_MARKER"; then
|
||||
rm -f -- "$temporary_marker"
|
||||
fail "could not create the Link reconciliation request"
|
||||
fi
|
||||
}
|
||||
|
||||
mesh_disable() {
|
||||
require_root
|
||||
systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
|
||||
|
|
@ -97,6 +119,7 @@ restore_project_firewall() {
|
|||
if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then
|
||||
systemctl enable "$FIREWALL_UNIT" >/dev/null
|
||||
fi
|
||||
request_reconciliation
|
||||
log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links."
|
||||
}
|
||||
|
||||
|
|
@ -108,6 +131,7 @@ mesh_enable() {
|
|||
fi
|
||||
systemctl enable "$WIREGUARD_UNIT" >/dev/null
|
||||
systemctl restart "$WIREGUARD_UNIT"
|
||||
request_reconciliation
|
||||
rm -f -- "$MARKER"
|
||||
log "Mesh enabled locally."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,33 @@
|
|||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
IFS=$'\n\t'
|
||||
export LC_ALL=C
|
||||
umask 077
|
||||
|
||||
readonly TABLE_FAMILY="inet"
|
||||
readonly TABLE_NAME="ochenstarik_smm"
|
||||
readonly CHAIN_NAME="links"
|
||||
|
||||
fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; }
|
||||
firewall_unavailable() { printf '%s\n' "mesh.firewall-unavailable" >&2; exit 79; }
|
||||
|
||||
testing="${SMM_POLICY_TESTING:-0}"
|
||||
if [[ "$testing" != "1" ]]; then
|
||||
[[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required"
|
||||
readonly STATE_FILE="/var/lib/ochenstarik-server-monitor-manager/mesh/nodes.tsv"
|
||||
readonly RECONCILE_MARKER="/var/lib/ochenstarik-server-monitor-manager/mesh/reconcile-requested"
|
||||
readonly RECONCILE_LOCK="/var/lib/ochenstarik-server-monitor-manager/mesh/reconcile-requested.lock"
|
||||
readonly FLOCK_COMMAND="/usr/bin/flock"
|
||||
else
|
||||
readonly STATE_FILE="${SMM_POLICY_STATE_FILE:?SMM_POLICY_STATE_FILE is required in testing mode}"
|
||||
readonly STATE_FILE="${SMM_POLICY_STATE_FILE:-/dev/null}"
|
||||
readonly RECONCILE_MARKER="${SMM_POLICY_RECONCILE_MARKER:-${TMPDIR:-/tmp}/smm-policy-reconcile-requested}"
|
||||
readonly RECONCILE_LOCK="${SMM_POLICY_RECONCILE_LOCK:-${RECONCILE_MARKER}.lock}"
|
||||
readonly FLOCK_COMMAND="${SMM_POLICY_FLOCK:-/usr/bin/flock}"
|
||||
fi
|
||||
|
||||
node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$'
|
||||
ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
|
||||
generation_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||
|
||||
validate_node_id() {
|
||||
[[ "$1" =~ $node_pattern ]] || fail "invalid node id"
|
||||
|
|
@ -59,20 +69,38 @@ run_nft() {
|
|||
fi
|
||||
}
|
||||
|
||||
inspect_firewall() {
|
||||
local listing
|
||||
if [[ "$testing" == "1" ]]; then
|
||||
[[ "${SMM_POLICY_FIREWALL_UNAVAILABLE:-0}" != "1" ]] || firewall_unavailable
|
||||
[[ -z "${SMM_POLICY_FIREWALL_ERROR:-}" ]] \
|
||||
|| fail "could not inspect nftables Link policy: $SMM_POLICY_FIREWALL_ERROR"
|
||||
return 0
|
||||
fi
|
||||
if listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" 2>&1)"; then
|
||||
printf '%s\n' "$listing"
|
||||
return 0
|
||||
fi
|
||||
if grep -Eiq 'No such file or directory|does not exist' <<<"$listing"; then
|
||||
firewall_unavailable
|
||||
fi
|
||||
fail "could not inspect nftables Link policy: ${listing%%$'\n'*}"
|
||||
}
|
||||
|
||||
ensure_firewall_available() {
|
||||
inspect_firewall >/dev/null
|
||||
}
|
||||
|
||||
rule_exists() {
|
||||
local comment="$1" listing
|
||||
if [[ "$testing" == "1" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if ! listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME")"; then
|
||||
fail "could not inspect nftables Link policy"
|
||||
fi
|
||||
listing="$(inspect_firewall)"
|
||||
grep -Fq "comment \"$comment\"" <<<"$listing"
|
||||
}
|
||||
|
||||
connect_rule() {
|
||||
local source_id="$1" target_id="$2" protocol="$3" port="$4"
|
||||
local source_ip target_ip comment
|
||||
ensure_firewall_available
|
||||
source_ip="$(lookup_node_ip "$source_id")"
|
||||
target_ip="$(lookup_node_ip "$target_id")"
|
||||
comment="smm:${source_id}:${target_id}:${protocol}:${port}"
|
||||
|
|
@ -84,6 +112,7 @@ connect_rule() {
|
|||
|
||||
disconnect_rule() {
|
||||
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle
|
||||
ensure_firewall_available
|
||||
# Resolve both identities before touching firewall state.
|
||||
lookup_node_ip "$source_id" >/dev/null
|
||||
lookup_node_ip "$target_id" >/dev/null
|
||||
|
|
@ -104,6 +133,7 @@ disconnect_rule() {
|
|||
|
||||
status_rule() {
|
||||
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment
|
||||
ensure_firewall_available
|
||||
# A status is meaningful only for identities currently known to the mesh.
|
||||
lookup_node_ip "$source_id" >/dev/null
|
||||
lookup_node_ip "$target_id" >/dev/null
|
||||
|
|
@ -115,7 +145,49 @@ status_rule() {
|
|||
fi
|
||||
}
|
||||
|
||||
reconcile_status() {
|
||||
local generation
|
||||
exec 9>"$RECONCILE_LOCK"
|
||||
chmod 0600 "$RECONCILE_LOCK"
|
||||
"$FLOCK_COMMAND" -x 9
|
||||
if [[ ! -f "$RECONCILE_MARKER" ]]; then
|
||||
printf '%s\n' complete
|
||||
return
|
||||
fi
|
||||
IFS= read -r generation <"$RECONCILE_MARKER" || fail "invalid reconciliation marker"
|
||||
[[ "$generation" =~ $generation_pattern ]] || fail "invalid reconciliation marker"
|
||||
printf 'requested:%s\n' "$generation"
|
||||
}
|
||||
|
||||
reconcile_complete() {
|
||||
local expected_generation="$1" current_generation=''
|
||||
[[ "$expected_generation" =~ $generation_pattern ]] || fail "invalid reconciliation generation"
|
||||
exec 9>"$RECONCILE_LOCK"
|
||||
chmod 0600 "$RECONCILE_LOCK"
|
||||
"$FLOCK_COMMAND" -x 9
|
||||
if [[ -f "$RECONCILE_MARKER" ]]; then
|
||||
IFS= read -r current_generation <"$RECONCILE_MARKER" || fail "invalid reconciliation marker"
|
||||
[[ "$current_generation" =~ $generation_pattern ]] || fail "invalid reconciliation marker"
|
||||
if [[ "$current_generation" == "$expected_generation" ]]; then
|
||||
rm -f -- "$RECONCILE_MARKER"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' complete
|
||||
}
|
||||
|
||||
action="${1:-}"
|
||||
case "$action" in
|
||||
reconcile-status)
|
||||
[[ $# -eq 1 ]] || fail "invalid reconcile-status argument count"
|
||||
reconcile_status
|
||||
exit 0
|
||||
;;
|
||||
reconcile-complete)
|
||||
[[ $# -eq 2 ]] || fail "invalid reconcile-complete argument count"
|
||||
reconcile_complete "$2"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
validate_rule "$@"
|
||||
case "$action" in
|
||||
link-connect) connect_rule "$2" "$3" "$4" "$5" ;;
|
||||
|
|
|
|||
|
|
@ -108,4 +108,4 @@ sudo ochenstarik-smm-emergency firewall-restore
|
|||
sudo ochenstarik-smm-emergency mesh-enable
|
||||
```
|
||||
|
||||
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub.
|
||||
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default и атомарно создаёт root-only запрос реконсиляции с уникальным generation. Control немедленно выполняет первый фоновый проход после старта, затем сверяет все действующие Link-политики не реже настроенного интервала `Control__LinkReconciliationSeconds` (по умолчанию 300 секунд). Новый marker запускает внеочередной проход на следующем poll tick, но не обходит backoff недоступного firewall. Helper удаляет только тот generation, который был прочитан перед успешно завершившимся проходом; более новый запрос сохраняется. При недоступной таблице запрос остаётся для retry с backoff, а Desktop показывает единый баннер «Mesh firewall не загружен». Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` также создаёт запрос реконсиляции; запускайте его только после проверки конфигурации и доступности Hub.
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@
|
|||
- [x] обязательное восстановление disabled policy после reconnect;
|
||||
- [x] append-only audit операций Link;
|
||||
- [x] интеграционные тесты kill switch, process restart и helper failure;
|
||||
- [x] B-2: независимая фоновая и emergency-triggered реконсиляция, агрегированное состояние недоступного Mesh firewall и Desktop banner;
|
||||
- [ ] B-3: развести результат реконсиляции на `Examined` / `Converged` / `Failed` (M2), выбрать фильтр или retention завершённых Disabled-политик (M4) и типизировать ещё не активированный Mesh Node (M5). Эти изменения вынесены отдельно, потому что меняют API/UI semantics и helper classification, тогда как B-2 ограничен восстановлением фактических правил и общим firewall failure;
|
||||
- [ ] выполнить физический acceptance Hub + source Node + два destination Node с WireGuard/nftables/reboot.
|
||||
|
||||
## Этап 4 — мониторинг и терминал
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ public sealed class ControlOptions
|
|||
|
||||
public int LinkExpirationPollSeconds { get; init; } = 15;
|
||||
|
||||
public int LinkReconciliationSeconds { get; init; } = 300;
|
||||
|
||||
public string BackupDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/backups";
|
||||
|
||||
public int BackupIntervalHours { get; init; } = 24;
|
||||
|
|
|
|||
|
|
@ -1353,6 +1353,17 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
|||
return link;
|
||||
}
|
||||
|
||||
public async Task<LinkPolicy?> GetLinkAsync(
|
||||
string id,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||
var link = await ReadLinkAsync(connection, transaction, id, cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return link;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<LinkPolicy>> ListLinksAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new List<LinkPolicy>();
|
||||
|
|
@ -1367,6 +1378,57 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
|||
return result;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<LinkPolicy>> ListEffectiveLinksAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new List<LinkPolicy>();
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
SELECT current.*
|
||||
FROM links AS current
|
||||
WHERE (current.desired_state = 'Active'
|
||||
OR (current.desired_state = 'Disabled' AND current.actual_state != 'Disabled'))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM links AS newer
|
||||
WHERE newer.source_node_id = current.source_node_id
|
||||
AND newer.target_node_id = current.target_node_id
|
||||
AND newer.protocol = current.protocol
|
||||
AND newer.port = current.port
|
||||
AND newer.version > current.version)
|
||||
ORDER BY current.version;
|
||||
""";
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
result.Add(ReadLink(reader));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<bool> IsEffectiveLinkAsync(
|
||||
LinkPolicy link,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
SELECT NOT EXISTS(
|
||||
SELECT 1 FROM links
|
||||
WHERE source_node_id = $source
|
||||
AND target_node_id = $target
|
||||
AND protocol = $protocol
|
||||
AND port = $port
|
||||
AND version > $version);
|
||||
""";
|
||||
command.Parameters.AddWithValue("$source", link.SourceNodeId);
|
||||
command.Parameters.AddWithValue("$target", link.TargetNodeId);
|
||||
command.Parameters.AddWithValue("$protocol", link.Protocol);
|
||||
command.Parameters.AddWithValue("$port", link.Port);
|
||||
command.Parameters.AddWithValue("$version", link.Version);
|
||||
return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)) == 1;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<LinkPolicy>> ListExpiredLinksAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken = default)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,17 @@ public interface ILinkPolicyApplier
|
|||
Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||
Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||
Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||
Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult<string?>(null);
|
||||
Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
public sealed class MeshFirewallUnavailableException : InvalidOperationException
|
||||
{
|
||||
public MeshFirewallUnavailableException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkPolicyApplier
|
||||
|
|
@ -55,6 +66,24 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
|
|||
};
|
||||
}
|
||||
|
||||
public async Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
|
||||
=> await RunAsync(["reconcile-status"], cancellationToken) switch
|
||||
{
|
||||
"complete" => null,
|
||||
var status when status.StartsWith("requested:", StringComparison.Ordinal)
|
||||
&& Guid.TryParseExact(status[10..], "D", out _) => status[10..],
|
||||
_ => throw new InvalidOperationException("Hub policy helper returned an invalid reconciliation marker status.")
|
||||
};
|
||||
|
||||
public async Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Guid.TryParseExact(generation, "D", out _))
|
||||
{
|
||||
throw new ArgumentException("Invalid reconciliation generation.", nameof(generation));
|
||||
}
|
||||
_ = await RunAsync(["reconcile-complete", generation], cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<string> RunAsync(IReadOnlyList<string> arguments, CancellationToken cancellationToken)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
|
|
@ -80,6 +109,11 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
|
|||
if (process.ExitCode != 0)
|
||||
{
|
||||
var message = (await error).Trim();
|
||||
if (process.ExitCode == 79
|
||||
&& string.Equals(message, LinkService.FirewallUnavailableCode, StringComparison.Ordinal))
|
||||
{
|
||||
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
||||
}
|
||||
throw new InvalidOperationException(string.IsNullOrWhiteSpace(message)
|
||||
? $"Hub policy helper exited with code {process.ExitCode}."
|
||||
: message);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ServerMonitorManager.Control;
|
||||
|
||||
public sealed class LinkReconciliationBackgroundService(
|
||||
LinkService links,
|
||||
ILinkPolicyApplier applier,
|
||||
IOptions<ControlOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<LinkReconciliationBackgroundService> logger) : BackgroundService
|
||||
{
|
||||
private readonly SemaphoreSlim _passGate = new(1, 1);
|
||||
private DateTimeOffset? _nextRegularAt;
|
||||
private DateTimeOffset? _backoffUntil;
|
||||
private int _unavailableAttempts;
|
||||
|
||||
internal async Task<LinkFullReconciliationResult?> RunOnceAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _passGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var now = timeProvider.GetUtcNow();
|
||||
string? requestGeneration = null;
|
||||
try
|
||||
{
|
||||
requestGeneration = await applier.GetReconciliationRequestAsync(cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not inspect the Link reconciliation marker.");
|
||||
}
|
||||
if (_backoffUntil is not null && now < _backoffUntil)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
if (requestGeneration is null && _nextRegularAt is not null && now < _nextRegularAt)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var result = await links.ReconcileAllAsync(cancellationToken);
|
||||
var interval = TimeSpan.FromSeconds(options.Value.LinkReconciliationSeconds);
|
||||
if (result.FirewallUnavailable)
|
||||
{
|
||||
_unavailableAttempts = Math.Min(_unavailableAttempts + 1, 4);
|
||||
_backoffUntil = now + TimeSpan.FromTicks(interval.Ticks * _unavailableAttempts);
|
||||
}
|
||||
else
|
||||
{
|
||||
_unavailableAttempts = 0;
|
||||
_backoffUntil = null;
|
||||
_nextRegularAt = now + interval;
|
||||
if (requestGeneration is not null && result.Failed == 0)
|
||||
{
|
||||
await applier.CompleteReconciliationAsync(requestGeneration, cancellationToken);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_passGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var pollSeconds = Math.Min(options.Value.LinkReconciliationSeconds, 30);
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(pollSeconds), timeProvider);
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await RunOnceAsync(stoppingToken);
|
||||
if (result is not null)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Link reconciliation completed: {Examined} examined, {Failed} failed, firewall unavailable: {Unavailable}.",
|
||||
result.Examined,
|
||||
result.Failed,
|
||||
result.FirewallUnavailable);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Link reconciliation failed.");
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,8 @@ public sealed class LinkService(
|
|||
ILinkPolicyApplier applier,
|
||||
ControlEventBroker events)
|
||||
{
|
||||
public const string FirewallUnavailableCode = "mesh.firewall-unavailable";
|
||||
public const string FirewallAvailableCode = "mesh.firewall-available";
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _reconciliationLocks = new();
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _nodeLocks = new();
|
||||
|
||||
|
|
@ -25,35 +27,20 @@ public sealed class LinkService(
|
|||
{
|
||||
return link;
|
||||
}
|
||||
var gate = _reconciliationLocks.GetOrAdd(link.Id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(link.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
var current = (await store.ListLinksAsync(cancellationToken))
|
||||
.SingleOrDefault(candidate => candidate.Id == link.Id)
|
||||
var current = await store.GetLinkAsync(link.Id, cancellationToken)
|
||||
?? throw new InvalidOperationException("The persisted Link disappeared.");
|
||||
if (current.DesiredState != "Active")
|
||||
if (current.DesiredState != "Active" || !await store.IsEffectiveLinkAsync(current, cancellationToken))
|
||||
{
|
||||
return current;
|
||||
}
|
||||
link = current;
|
||||
Publish("link.connecting", link);
|
||||
try
|
||||
if (!mutation.IsReplay)
|
||||
{
|
||||
await applier.ApplyConnectAsync(link, cancellationToken);
|
||||
await VerifyFactualStateAsync(link, expectedConnected: true, cancellationToken);
|
||||
link = await store.SetLinkActualStateAsync(link.Id, "Active", null, actor, cancellationToken)
|
||||
?? throw new InvalidOperationException("The persisted Link disappeared.");
|
||||
Publish("link.active", link);
|
||||
Publish("link.connecting", current);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
link = await store.SetLinkActualStateAsync(
|
||||
link.Id, "Failed", CompactError(exception), actor, cancellationToken)
|
||||
?? link;
|
||||
Publish("link.failed", link);
|
||||
}
|
||||
return link;
|
||||
return await ConvergeAsync(current, expectedConnected: true, actor, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
|
@ -67,8 +54,7 @@ public sealed class LinkService(
|
|||
string actor,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = _reconciliationLocks.GetOrAdd(id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(id, cancellationToken);
|
||||
try
|
||||
{
|
||||
return await DisableCoreAsync(id, request, actor, cancellationToken);
|
||||
|
|
@ -99,7 +85,7 @@ public sealed class LinkService(
|
|||
{
|
||||
Publish("link.disconnecting", link);
|
||||
}
|
||||
return await ConvergeDisabledCoreAsync(link, actor, cancellationToken);
|
||||
return await ConvergeAsync(link, expectedConnected: false, actor, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<LinkPolicy> ConvergeDisabledAsync(
|
||||
|
|
@ -107,11 +93,10 @@ public sealed class LinkService(
|
|||
string actor,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = _reconciliationLocks.GetOrAdd(link.Id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(link.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
return await ConvergeDisabledCoreAsync(link, actor, cancellationToken);
|
||||
return await ConvergeAsync(link, expectedConnected: false, actor, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
|
@ -130,54 +115,24 @@ public sealed class LinkService(
|
|||
{
|
||||
using var nodeLease = await AcquireNodeLocksAsync(
|
||||
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
||||
var gate = _reconciliationLocks.GetOrAdd(candidate.Id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
var current = (await store.ListEffectiveLinksForNodeAsync(nodeId, cancellationToken))
|
||||
.SingleOrDefault(link => link.Id == candidate.Id);
|
||||
if (current is null)
|
||||
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
|
||||
if (current is null || (current.SourceNodeId != nodeId && current.TargetNodeId != nodeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var actor = $"system:reconnect:{nodeId}";
|
||||
var expectedConnected = current.DesiredState == "Active";
|
||||
var pendingState = expectedConnected ? "Connecting" : "Disconnecting";
|
||||
var completedState = expectedConnected ? "Active" : "Disabled";
|
||||
var completedEvent = expectedConnected ? "link.active" : "link.disabled";
|
||||
var failureState = expectedConnected ? "Failed" : "Partial";
|
||||
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
|
||||
var link = await store.SetLinkActualStateAsync(
|
||||
current.Id, pendingState, null, actor, cancellationToken) ?? current;
|
||||
Publish("link.reconciling", link);
|
||||
try
|
||||
var result = await ConvergeAsync(
|
||||
current,
|
||||
current.DesiredState == "Active",
|
||||
$"system:reconnect:{nodeId}",
|
||||
cancellationToken);
|
||||
reconciled++;
|
||||
if (result.ActualState is "Failed" or "Partial")
|
||||
{
|
||||
var isConnected = await applier.IsConnectedAsync(link, cancellationToken);
|
||||
if (isConnected != expectedConnected)
|
||||
{
|
||||
if (expectedConnected)
|
||||
{
|
||||
await applier.ApplyConnectAsync(link, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await applier.ApplyDisconnectAsync(link, cancellationToken);
|
||||
}
|
||||
await VerifyFactualStateAsync(link, expectedConnected, cancellationToken);
|
||||
}
|
||||
link = await store.SetLinkActualStateAsync(
|
||||
link.Id, completedState, null, actor, cancellationToken) ?? link;
|
||||
Publish(completedEvent, link);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
link = await store.SetLinkActualStateAsync(
|
||||
link.Id, failureState, CompactError(exception), actor, cancellationToken) ?? link;
|
||||
Publish(failureEvent, link);
|
||||
failed++;
|
||||
}
|
||||
reconciled++;
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
|
@ -187,6 +142,69 @@ public sealed class LinkService(
|
|||
return new LinkReconciliationResult(reconciled, failed);
|
||||
}
|
||||
|
||||
public async Task<LinkFullReconciliationResult> ReconcileAllAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var candidates = await store.ListEffectiveLinksAsync(cancellationToken);
|
||||
var recoveringFirewall = candidates.Any(candidate =>
|
||||
string.Equals(candidate.LastError, FirewallUnavailableCode, StringComparison.Ordinal));
|
||||
var examined = 0;
|
||||
var failed = 0;
|
||||
var firewallUnavailable = false;
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
using var nodeLease = await AcquireNodeLocksAsync(
|
||||
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
|
||||
if (current is null || !IsEligibleForFullReconciliation(current))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var result = await ConvergeAsync(
|
||||
current, current.DesiredState == "Active", "system:reconcile", cancellationToken);
|
||||
examined++;
|
||||
if (result.LastError == FirewallUnavailableCode)
|
||||
{
|
||||
firewallUnavailable = true;
|
||||
}
|
||||
else if (result.ActualState is "Failed" or "Partial")
|
||||
{
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
if (firewallUnavailable)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (firewallUnavailable)
|
||||
{
|
||||
await MarkFirewallUnavailableAsync(candidates, cancellationToken);
|
||||
events.Publish(
|
||||
FirewallUnavailableCode,
|
||||
"mesh",
|
||||
JsonSerializer.Serialize(
|
||||
new ControlError(FirewallUnavailableCode), SmmJsonContext.Default.ControlError));
|
||||
return new LinkFullReconciliationResult(examined, candidates.Count, true);
|
||||
}
|
||||
if (recoveringFirewall)
|
||||
{
|
||||
events.Publish(
|
||||
FirewallAvailableCode,
|
||||
"mesh",
|
||||
JsonSerializer.Serialize(
|
||||
new ControlError(FirewallAvailableCode), SmmJsonContext.Default.ControlError));
|
||||
}
|
||||
return new LinkFullReconciliationResult(examined, failed, false);
|
||||
}
|
||||
|
||||
public async Task<LinkExpirationResult> ExpireDueLinksAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
|
|
@ -196,24 +214,30 @@ public sealed class LinkService(
|
|||
var links = await store.ListExpiredLinksAsync(now, cancellationToken);
|
||||
foreach (var candidate in links)
|
||||
{
|
||||
var gate = _reconciliationLocks.GetOrAdd(candidate.Id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
var current = (await store.ListExpiredLinksAsync(now, cancellationToken))
|
||||
.SingleOrDefault(link => link.Id == candidate.Id);
|
||||
if (current is null)
|
||||
var current = await store.GetLinkAsync(candidate.Id, cancellationToken);
|
||||
if (current is null || current.ExpiresAt is null || current.ExpiresAt > now
|
||||
|| !await store.IsEffectiveLinkAsync(current, cancellationToken))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
LinkPolicy? result;
|
||||
if (current.DesiredState == "Active")
|
||||
{
|
||||
var result = await DisableCoreAsync(
|
||||
result = await DisableCoreAsync(
|
||||
current.Id,
|
||||
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
|
||||
"system:ttl",
|
||||
cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
Publish("link.disconnecting", current);
|
||||
result = await ConvergeAsync(
|
||||
current, expectedConnected: false, "system:ttl-retry", cancellationToken);
|
||||
}
|
||||
if (result?.ActualState == "Disabled")
|
||||
{
|
||||
disabled++;
|
||||
|
|
@ -222,29 +246,6 @@ public sealed class LinkService(
|
|||
{
|
||||
failed++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var retrying = await store.SetLinkActualStateAsync(
|
||||
current.Id, "Disconnecting", null, "system:ttl-retry", cancellationToken) ?? current;
|
||||
Publish("link.disconnecting", retrying);
|
||||
try
|
||||
{
|
||||
await applier.ApplyDisconnectAsync(retrying, cancellationToken);
|
||||
await VerifyFactualStateAsync(retrying, expectedConnected: false, cancellationToken);
|
||||
var completed = await store.SetLinkActualStateAsync(
|
||||
retrying.Id, "Disabled", null, "system:ttl-retry", cancellationToken) ?? retrying;
|
||||
Publish("link.disabled", completed);
|
||||
disabled++;
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
var partial = await store.SetLinkActualStateAsync(
|
||||
retrying.Id, "Partial", CompactError(exception), "system:ttl-retry", cancellationToken)
|
||||
?? retrying;
|
||||
Publish("link.partial", partial);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
|
@ -254,6 +255,119 @@ public sealed class LinkService(
|
|||
return new LinkExpirationResult(disabled, failed);
|
||||
}
|
||||
|
||||
private async Task<LinkPolicy> ConvergeAsync(
|
||||
LinkPolicy link,
|
||||
bool expectedConnected,
|
||||
string actor,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await store.GetLinkAsync(link.Id, cancellationToken) ?? link;
|
||||
if ((current.DesiredState == "Active") != expectedConnected
|
||||
|| !await store.IsEffectiveLinkAsync(current, cancellationToken))
|
||||
{
|
||||
return current;
|
||||
}
|
||||
|
||||
var completedState = expectedConnected ? "Active" : "Disabled";
|
||||
var failureState = expectedConnected ? "Failed" : "Partial";
|
||||
var completedEvent = expectedConnected ? "link.active" : "link.disabled";
|
||||
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
|
||||
try
|
||||
{
|
||||
var isConnected = await applier.IsConnectedAsync(current, cancellationToken);
|
||||
var changedFact = isConnected != expectedConnected;
|
||||
if (changedFact)
|
||||
{
|
||||
var pendingState = expectedConnected ? "Connecting" : "Disconnecting";
|
||||
if (current.ActualState != pendingState || current.LastError is not null)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, pendingState, null, actor, cancellationToken) ?? current;
|
||||
}
|
||||
if (actor.StartsWith("system:", StringComparison.Ordinal))
|
||||
{
|
||||
Publish("link.reconciling", current);
|
||||
}
|
||||
if (expectedConnected)
|
||||
{
|
||||
await applier.ApplyConnectAsync(current, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
await applier.ApplyDisconnectAsync(current, cancellationToken);
|
||||
}
|
||||
await VerifyFactualStateAsync(current, expectedConnected, cancellationToken);
|
||||
}
|
||||
if (current.ActualState != completedState || current.LastError is not null)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, completedState, null, actor, cancellationToken) ?? current;
|
||||
Publish(completedEvent, current);
|
||||
}
|
||||
if (changedFact && actor == "system:reconcile")
|
||||
{
|
||||
Publish(expectedConnected ? "link.reapplied" : "link.orphan-removed", current);
|
||||
}
|
||||
}
|
||||
catch (MeshFirewallUnavailableException)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, failureState, FirewallUnavailableCode, actor, cancellationToken) ?? current;
|
||||
if (actor != "system:reconcile")
|
||||
{
|
||||
Publish(failureEvent, current);
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, failureState, CompactError(exception), actor, cancellationToken) ?? current;
|
||||
Publish(failureEvent, current);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private async Task<LinkPolicy?> GetCurrentEffectiveAsync(
|
||||
string id,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await store.GetLinkAsync(id, cancellationToken);
|
||||
if (current is null || !await store.IsEffectiveLinkAsync(current, cancellationToken))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private async Task MarkFirewallUnavailableAsync(
|
||||
IReadOnlyList<LinkPolicy> candidates,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
using var nodeLease = await AcquireNodeLocksAsync(
|
||||
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
||||
try
|
||||
{
|
||||
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
|
||||
if (current is not null && IsEligibleForFullReconciliation(current))
|
||||
{
|
||||
await store.SetLinkActualStateAsync(
|
||||
current.Id, "Partial", FirewallUnavailableCode, "system:reconcile", cancellationToken);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsEligibleForFullReconciliation(LinkPolicy link)
|
||||
=> link.DesiredState == "Active"
|
||||
|| (link.DesiredState == "Disabled" && link.ActualState != "Disabled");
|
||||
|
||||
private void Publish(string type, LinkPolicy link)
|
||||
=> events.Publish(
|
||||
type,
|
||||
|
|
@ -289,38 +403,13 @@ public sealed class LinkService(
|
|||
}
|
||||
}
|
||||
|
||||
private async Task<LinkPolicy> ConvergeDisabledCoreAsync(
|
||||
LinkPolicy link,
|
||||
string actor,
|
||||
private async Task<SemaphoreSlim> AcquireLinkGateAsync(
|
||||
string id,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = (await store.ListLinksAsync(cancellationToken))
|
||||
.SingleOrDefault(candidate => candidate.Id == link.Id) ?? link;
|
||||
if (current.DesiredState != "Disabled")
|
||||
{
|
||||
return current;
|
||||
}
|
||||
try
|
||||
{
|
||||
if (await applier.IsConnectedAsync(current, cancellationToken))
|
||||
{
|
||||
await applier.ApplyDisconnectAsync(current, cancellationToken);
|
||||
await VerifyFactualStateAsync(current, expectedConnected: false, cancellationToken);
|
||||
}
|
||||
if (current.ActualState != "Disabled" || current.LastError is not null)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, "Disabled", null, actor, cancellationToken) ?? current;
|
||||
Publish("link.disabled", current);
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
current = await store.SetLinkActualStateAsync(
|
||||
current.Id, "Partial", CompactError(exception), actor, cancellationToken) ?? current;
|
||||
Publish("link.partial", current);
|
||||
}
|
||||
return current;
|
||||
var gate = _reconciliationLocks.GetOrAdd(id, static _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
return gate;
|
||||
}
|
||||
|
||||
private async Task VerifyFactualStateAsync(
|
||||
|
|
@ -336,7 +425,7 @@ public sealed class LinkService(
|
|||
}
|
||||
|
||||
private static string CompactError(Exception exception)
|
||||
=> exception.Message.Split([(char)13, '\n'], StringSplitOptions.RemoveEmptyEntries)
|
||||
=> exception.Message.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
|
||||
.FirstOrDefault() ?? "Policy application failed.";
|
||||
|
||||
private sealed class LockLease(SemaphoreSlim[] gates) : IDisposable
|
||||
|
|
@ -352,3 +441,4 @@ public sealed class LinkService(
|
|||
}
|
||||
|
||||
public sealed record LinkExpirationResult(int Disabled, int Failed);
|
||||
public sealed record LinkFullReconciliationResult(int Examined, int Failed, bool FirewallUnavailable);
|
||||
|
|
|
|||
|
|
@ -46,9 +46,10 @@ builder.Services.AddOptions<ControlOptions>()
|
|||
&& options.AuditRetentionDays is >= 1 and <= 3650
|
||||
&& options.MaintenanceIntervalMinutes is >= 1 and <= 1440
|
||||
&& options.LinkExpirationPollSeconds is >= 1 and <= 300
|
||||
&& options.LinkReconciliationSeconds is >= 30 and <= 3600
|
||||
&& options.BackupIntervalHours is >= 1 and <= 720
|
||||
&& options.BackupRetentionCount is >= 1 and <= 100,
|
||||
"Invalid Control paths, heartbeat, retention, maintenance, expiration, or backup settings.")
|
||||
"Invalid Control paths, heartbeat, retention, maintenance, expiration, reconciliation, or backup settings.")
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
builder.Services.AddSingleton<ControlStore>();
|
||||
|
|
@ -59,6 +60,7 @@ builder.Services.AddSingleton<LinkService>();
|
|||
builder.Services.AddSingleton<CertificateLifecycleService>();
|
||||
builder.Services.AddSingleton<ControlBackupService>();
|
||||
builder.Services.AddHostedService<LinkExpirationBackgroundService>();
|
||||
builder.Services.AddHostedService<LinkReconciliationBackgroundService>();
|
||||
builder.Services.AddHostedService<ControlMaintenanceBackgroundService>();
|
||||
builder.Services.AddAuthentication(CertificateAuthenticationDefaults.AuthenticationScheme)
|
||||
.AddCertificate(options =>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"AuditRetentionDays": 90,
|
||||
"MaintenanceIntervalMinutes": 15,
|
||||
"LinkExpirationPollSeconds": 15,
|
||||
"LinkReconciliationSeconds": 300,
|
||||
"BackupDirectory": "/var/lib/ochenstarik-server-monitor-manager/backups",
|
||||
"BackupIntervalHours": 24,
|
||||
"BackupRetentionCount": 7,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ public sealed partial class MainPage : Page
|
|||
private SettingsPage? _settingsPage;
|
||||
private bool _loaded;
|
||||
private bool _controlListening;
|
||||
private bool _meshFirewallUnavailable;
|
||||
|
||||
public MainPage()
|
||||
{
|
||||
|
|
@ -160,7 +161,7 @@ public sealed partial class MainPage : Page
|
|||
InfoBarSeverity.Warning);
|
||||
}
|
||||
|
||||
if (Servers.Count > 0)
|
||||
if (Servers.Count > 0 || _control.IsConfigured)
|
||||
{
|
||||
await RefreshAllAsync();
|
||||
}
|
||||
|
|
@ -449,6 +450,18 @@ public sealed partial class MainPage : Page
|
|||
{
|
||||
DispatcherQueue.TryEnqueue(() =>
|
||||
{
|
||||
if (controlEvent.Type == MeshLinkViewModel.FirewallUnavailableErrorCode)
|
||||
{
|
||||
SetMeshFirewallUnavailable(true);
|
||||
_ = RefreshMeshAsync(showSuccess: false);
|
||||
return;
|
||||
}
|
||||
if (controlEvent.Type == "mesh.firewall-available")
|
||||
{
|
||||
SetMeshFirewallUnavailable(false);
|
||||
_ = RefreshMeshAsync(showSuccess: false);
|
||||
return;
|
||||
}
|
||||
if (controlEvent.Type.StartsWith("link.", StringComparison.Ordinal))
|
||||
{
|
||||
ShowInfo(
|
||||
|
|
@ -730,6 +743,11 @@ public sealed partial class MainPage : Page
|
|||
{
|
||||
if (Servers.Count == 0)
|
||||
{
|
||||
if (_control.IsConfigured)
|
||||
{
|
||||
await RefreshControlMeshAsync(showSuccess: false);
|
||||
return;
|
||||
}
|
||||
ShowInfo("Серверы не добавлены", "Сначала создайте SSH-ключ и установите его на сервере.", InfoBarSeverity.Informational);
|
||||
return;
|
||||
}
|
||||
|
|
@ -916,6 +934,12 @@ public sealed partial class MainPage : Page
|
|||
LinkActionInfo.IsOpen = true;
|
||||
}
|
||||
|
||||
private void SetMeshFirewallUnavailable(bool unavailable)
|
||||
{
|
||||
_meshFirewallUnavailable = unavailable;
|
||||
_linksPage?.SetFirewallUnavailable(unavailable);
|
||||
}
|
||||
|
||||
private ServerViewModel? FindHub()
|
||||
=> Servers.FirstOrDefault(server => server.IsHub);
|
||||
|
||||
|
|
@ -1031,6 +1055,11 @@ public sealed partial class MainPage : Page
|
|||
link.LastError));
|
||||
}
|
||||
|
||||
SetMeshFirewallUnavailable(links
|
||||
.GroupBy(link => new { link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port })
|
||||
.Select(group => group.MaxBy(link => link.Version)!)
|
||||
.Any(link => link.LastError == MeshLinkViewModel.FirewallUnavailableErrorCode));
|
||||
|
||||
var activeLinks = MeshLinks.Count(link => link.ActualState == "Active");
|
||||
ActiveLinksValueText.Text = activeLinks.ToString(CultureInfo.InvariantCulture);
|
||||
MeshStatusText.Text = $"Control · {MeshNodes.Count} узлов · {activeLinks} активных · {MeshLinks.Count} политик";
|
||||
|
|
@ -1289,6 +1318,7 @@ public sealed partial class MainPage : Page
|
|||
break;
|
||||
case "links":
|
||||
_linksPage ??= new LinksPage(this);
|
||||
_linksPage.SetFirewallUnavailable(_meshFirewallUnavailable);
|
||||
ShowNavigationPage(_linksPage);
|
||||
break;
|
||||
case "sessions":
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ public sealed class MeshNodeViewModel
|
|||
|
||||
public sealed class MeshLinkViewModel
|
||||
{
|
||||
public const string FirewallUnavailableErrorCode = "mesh.firewall-unavailable";
|
||||
|
||||
public MeshLinkViewModel(
|
||||
string source,
|
||||
string target,
|
||||
|
|
@ -63,7 +65,8 @@ public sealed class MeshLinkViewModel
|
|||
public string DesiredStatusText => $"Желаемое состояние: {DesiredState}";
|
||||
public string ActualStatusText => $"Фактическое состояние: {ActualState}";
|
||||
public string DriftText => HasDrift ? "Расхождение: требуется сверка политики" : "Расхождение: нет";
|
||||
public string ErrorText => string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}";
|
||||
public string ErrorText => LastError == FirewallUnavailableErrorCode ? string.Empty
|
||||
: string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}";
|
||||
public string VersionText => $"Версия политики: {Version}";
|
||||
public string ExpirationText => ExpiresUnix == 0
|
||||
? "вручную"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
<Grid RowSpacing="16">
|
||||
<Grid.RowDefinitions>
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="Auto" />
|
||||
<RowDefinition Height="*" />
|
||||
</Grid.RowDefinitions>
|
||||
|
|
@ -27,7 +28,15 @@
|
|||
</CommandBar>
|
||||
</Grid>
|
||||
|
||||
<Grid x:Name="LinksWorkspace" Grid.Row="1" ColumnSpacing="20" RowSpacing="20">
|
||||
<InfoBar
|
||||
x:Name="FirewallUnavailableInfo"
|
||||
Grid.Row="1"
|
||||
IsClosable="False"
|
||||
IsOpen="False"
|
||||
Message="Mesh firewall не загружен"
|
||||
Severity="Error" />
|
||||
|
||||
<Grid x:Name="LinksWorkspace" Grid.Row="2" ColumnSpacing="20" RowSpacing="20">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition x:Name="EditorColumn" Width="*" />
|
||||
<ColumnDefinition x:Name="ListColumn" Width="0" />
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ public sealed partial class LinksPage : Page
|
|||
|
||||
public ObservableCollection<MeshLinkViewModel> Links => _host.MeshLinks;
|
||||
|
||||
internal void SetFirewallUnavailable(bool unavailable)
|
||||
=> FirewallUnavailableInfo.IsOpen = unavailable;
|
||||
|
||||
private async void RefreshButton_Click(object sender, RoutedEventArgs e)
|
||||
=> await _host.RefreshLinksFromPageAsync();
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
|||
var sudoPath = Path.Combine(_directory, "sudo");
|
||||
var helperPath = Path.Combine(_directory, "policy-helper");
|
||||
var failureMarkerPath = Path.Combine(_directory, "fail-disconnect");
|
||||
var firewallUnavailableMarkerPath = Path.Combine(_directory, "firewall-unavailable");
|
||||
var connectedMarkerPath = Path.Combine(_directory, "connected");
|
||||
var invocationLogPath = Path.Combine(_directory, "helper.log");
|
||||
await WriteExecutableAsync(
|
||||
|
|
@ -58,6 +59,10 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
|||
rm -f '{{ShellQuote(connectedMarkerPath)}}'
|
||||
fi
|
||||
if [ "${1:-}" = "link-status" ]; then
|
||||
if [ -f '{{ShellQuote(firewallUnavailableMarkerPath)}}' ]; then
|
||||
echo "mesh.firewall-unavailable" >&2
|
||||
exit 79
|
||||
fi
|
||||
if [ -f '{{ShellQuote(connectedMarkerPath)}}' ]; then
|
||||
printf '%s\n' active
|
||||
else
|
||||
|
|
@ -79,6 +84,28 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
|||
cancellationToken);
|
||||
Assert.Equal("Active", active.ActualState);
|
||||
|
||||
File.Delete(connectedMarkerPath);
|
||||
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||
var restored = await service.ReconcileAllAsync(cancellationToken);
|
||||
Assert.False(restored.FirewallUnavailable);
|
||||
Assert.Equal("Active", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
||||
Assert.Equal(
|
||||
[
|
||||
"link-status ai-agent home tcp 22",
|
||||
"link-connect ai-agent home tcp 22 60",
|
||||
"link-status ai-agent home tcp 22"
|
||||
],
|
||||
await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
||||
|
||||
await File.WriteAllTextAsync(firewallUnavailableMarkerPath, "fail", cancellationToken);
|
||||
var unavailable = await service.ReconcileAllAsync(cancellationToken);
|
||||
Assert.True(unavailable.FirewallUnavailable);
|
||||
Assert.Equal(LinkService.FirewallUnavailableCode,
|
||||
(await store.GetLinkAsync(active.Id, cancellationToken))!.LastError);
|
||||
File.Delete(firewallUnavailableMarkerPath);
|
||||
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
|
||||
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||
|
||||
await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken);
|
||||
var partial = await service.DisableAsync(
|
||||
active.Id,
|
||||
|
|
@ -110,16 +137,14 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
|||
Assert.Equal("Disabled", persisted.ActualState);
|
||||
|
||||
var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken);
|
||||
Assert.Equal(9, invocations.Length);
|
||||
Assert.Equal("link-connect ai-agent home tcp 22 60", invocations[0]);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[1]);
|
||||
Assert.Equal(7, invocations.Length);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[0]);
|
||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[1]);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[2]);
|
||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[3]);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[4]);
|
||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[5]);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[6]);
|
||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[7]);
|
||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[8]);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,338 @@
|
|||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ServerMonitorManager.Control;
|
||||
using ServerMonitorManager.Core;
|
||||
using Xunit;
|
||||
|
||||
namespace ServerMonitorManager.Control.Tests;
|
||||
|
||||
public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(
|
||||
Path.GetTempPath(), $"smm-link-reconciliation-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public async Task AllPolicyPassReappliesErasedActiveRulesWithoutHeartbeat()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "AA11", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "BB22", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-two", "CC33", cancellationToken);
|
||||
var applier = new RecordingPolicyApplier();
|
||||
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||
var first = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
var second = await service.CreateAsync(CreateRequest("target-two"), "operator", cancellationToken);
|
||||
applier.EraseRules();
|
||||
var mutationsBefore = applier.ConnectCalls;
|
||||
|
||||
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||
|
||||
Assert.Equal(2, result.Examined);
|
||||
Assert.Equal(0, result.Failed);
|
||||
Assert.Equal(mutationsBefore + 2, applier.ConnectCalls);
|
||||
Assert.All(await store.ListEffectiveLinksAsync(cancellationToken),
|
||||
link => Assert.Equal("Active", link.ActualState));
|
||||
Assert.True(applier.IsConnected(first));
|
||||
Assert.True(applier.IsConnected(second));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecondUnchangedPassDoesNotMutateFirewall()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "A111", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "B222", cancellationToken);
|
||||
var applier = new RecordingPolicyApplier();
|
||||
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||
await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
|
||||
await service.ReconcileAllAsync(cancellationToken);
|
||||
var connects = applier.ConnectCalls;
|
||||
var disconnects = applier.DisconnectCalls;
|
||||
await service.ReconcileAllAsync(cancellationToken);
|
||||
|
||||
Assert.Equal(connects, applier.ConnectCalls);
|
||||
Assert.Equal(disconnects, applier.DisconnectCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllPolicyPassRemovesOrphanRuleForDesiredDisabledLink()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "DD44", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "EE55", cancellationToken);
|
||||
var broker = new ControlEventBroker();
|
||||
using var subscription = broker.Subscribe();
|
||||
var applier = new RecordingPolicyApplier();
|
||||
var service = new LinkService(store, applier, broker);
|
||||
var active = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
await store.BeginDisableLinkMutationAsync(
|
||||
active.Id,
|
||||
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
|
||||
"operator",
|
||||
cancellationToken);
|
||||
|
||||
await service.ReconcileAllAsync(cancellationToken);
|
||||
|
||||
Assert.Equal(1, applier.DisconnectCalls);
|
||||
Assert.Equal("Disabled", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
||||
var eventTypes = new List<string>();
|
||||
while (subscription.Reader.TryRead(out var controlEvent))
|
||||
{
|
||||
eventTypes.Add(controlEvent.Type);
|
||||
}
|
||||
Assert.Contains("link.orphan-removed", eventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FirewallUnavailableAbortsWithoutMutationsAndPublishesOneSharedEvent()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "FF66", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "0011", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-two", "0022", cancellationToken);
|
||||
var broker = new ControlEventBroker();
|
||||
using var subscription = broker.Subscribe();
|
||||
var applier = new RecordingPolicyApplier();
|
||||
var service = new LinkService(store, applier, broker);
|
||||
await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
await service.CreateAsync(CreateRequest("target-two"), "operator", cancellationToken);
|
||||
while (subscription.Reader.TryRead(out _)) { }
|
||||
var connectBefore = applier.ConnectCalls;
|
||||
var disconnectBefore = applier.DisconnectCalls;
|
||||
applier.FirewallUnavailable = true;
|
||||
|
||||
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||
|
||||
Assert.True(result.FirewallUnavailable);
|
||||
Assert.Equal(connectBefore, applier.ConnectCalls);
|
||||
Assert.Equal(disconnectBefore, applier.DisconnectCalls);
|
||||
Assert.All(await store.ListEffectiveLinksAsync(cancellationToken), link =>
|
||||
{
|
||||
Assert.Equal("Partial", link.ActualState);
|
||||
Assert.Equal(LinkService.FirewallUnavailableCode, link.LastError);
|
||||
});
|
||||
var eventTypes = new List<string>();
|
||||
while (subscription.Reader.TryRead(out var controlEvent))
|
||||
{
|
||||
eventTypes.Add(controlEvent.Type);
|
||||
}
|
||||
Assert.Equal([LinkService.FirewallUnavailableCode], eventTypes);
|
||||
|
||||
applier.FirewallUnavailable = false;
|
||||
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
|
||||
eventTypes.Clear();
|
||||
while (subscription.Reader.TryRead(out var controlEvent))
|
||||
{
|
||||
eventTypes.Add(controlEvent.Type);
|
||||
}
|
||||
Assert.Single(eventTypes, eventType => eventType == LinkService.FirewallAvailableCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkerCreatedAfterNormalPassTriggersPromptAdditionalPass()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "A333", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "B444", cancellationToken);
|
||||
var applier = new RecordingPolicyApplier();
|
||||
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
var background = CreateBackgroundService(
|
||||
links,
|
||||
applier,
|
||||
new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z")));
|
||||
|
||||
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||
Assert.Equal(0, applier.CompleteReconciliationCalls);
|
||||
applier.ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
|
||||
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||
Assert.Equal(2, applier.ReconciliationStatusCalls);
|
||||
Assert.Equal(1, applier.CompleteReconciliationCalls);
|
||||
Assert.Null(applier.ReconciliationRequest);
|
||||
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletingObservedGenerationDoesNotConsumeNewerRequest()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "A355", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "B466", cancellationToken);
|
||||
var generationA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
var generationB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
|
||||
var applier = new RecordingPolicyApplier { ReconciliationRequest = generationA };
|
||||
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
applier.ReplacementRequestOnNextProbe = generationB;
|
||||
var background = CreateBackgroundService(
|
||||
links,
|
||||
applier,
|
||||
new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z")));
|
||||
|
||||
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||
Assert.Equal([generationA], applier.CompletedGenerations);
|
||||
Assert.Equal(generationB, applier.ReconciliationRequest);
|
||||
|
||||
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||
Assert.Equal([generationA, generationB], applier.CompletedGenerations);
|
||||
Assert.Null(applier.ReconciliationRequest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BackgroundPassRetainsMarkerWhenFirewallIsUnavailable()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
await EnrollAgentAsync(store, "source", "A555", cancellationToken);
|
||||
await EnrollAgentAsync(store, "target-one", "B666", cancellationToken);
|
||||
var applier = new RecordingPolicyApplier
|
||||
{
|
||||
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||||
};
|
||||
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||
applier.FirewallUnavailable = true;
|
||||
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z"));
|
||||
var background = CreateBackgroundService(links, applier, time);
|
||||
|
||||
var result = await background.RunOnceAsync(cancellationToken);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.True(result.FirewallUnavailable);
|
||||
Assert.Equal(0, applier.CompleteReconciliationCalls);
|
||||
Assert.NotNull(applier.ReconciliationRequest);
|
||||
applier.FirewallUnavailable = false;
|
||||
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||
time.Advance(TimeSpan.FromSeconds(30));
|
||||
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||
Assert.Null(applier.ReconciliationRequest);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
if (Directory.Exists(_directory))
|
||||
{
|
||||
Directory.Delete(_directory, recursive: true);
|
||||
}
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private ControlStore CreateStore()
|
||||
{
|
||||
Directory.CreateDirectory(_directory);
|
||||
return new ControlStore(Options.Create(new ControlOptions
|
||||
{
|
||||
DatabasePath = Path.Combine(_directory, "control.db"),
|
||||
CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx")
|
||||
}));
|
||||
}
|
||||
|
||||
private static LinkPolicyCreateRequest CreateRequest(string target)
|
||||
=> new("source", target, "tcp", 22, 0, "test", Guid.NewGuid().ToString());
|
||||
|
||||
private static LinkReconciliationBackgroundService CreateBackgroundService(
|
||||
LinkService links,
|
||||
ILinkPolicyApplier applier,
|
||||
TimeProvider timeProvider)
|
||||
=> new(
|
||||
links,
|
||||
applier,
|
||||
Options.Create(new ControlOptions { LinkReconciliationSeconds = 30 }),
|
||||
timeProvider,
|
||||
NullLogger<LinkReconciliationBackgroundService>.Instance);
|
||||
|
||||
private static async Task EnrollAgentAsync(
|
||||
ControlStore store,
|
||||
string nodeId,
|
||||
string thumbprint,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var token = await store.CreateEnrollmentTokenAsync(nodeId, TimeSpan.FromMinutes(10), cancellationToken);
|
||||
Assert.NotNull(await store.EnrollAsync(
|
||||
new EnrollmentRequest(nodeId, token, "csr", Guid.NewGuid().ToString()),
|
||||
() => new IssuedCertificate("certificate", "ca", thumbprint, DateTimeOffset.UtcNow.AddYears(1)),
|
||||
cancellationToken));
|
||||
}
|
||||
|
||||
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
||||
{
|
||||
private readonly HashSet<string> _connected = [];
|
||||
public int ConnectCalls { get; private set; }
|
||||
public int DisconnectCalls { get; private set; }
|
||||
public bool FirewallUnavailable { get; set; }
|
||||
public string? ReconciliationRequest { get; set; }
|
||||
public string? ReplacementRequestOnNextProbe { get; set; }
|
||||
public int ReconciliationStatusCalls { get; private set; }
|
||||
public int CompleteReconciliationCalls { get; private set; }
|
||||
public List<string> CompletedGenerations { get; } = [];
|
||||
|
||||
public void EraseRules() => _connected.Clear();
|
||||
public bool IsConnected(LinkPolicy link) => _connected.Contains(link.Id);
|
||||
|
||||
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||
{
|
||||
ConnectCalls++;
|
||||
_connected.Add(link.Id);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||
{
|
||||
DisconnectCalls++;
|
||||
_connected.Remove(link.Id);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||
{
|
||||
if (ReplacementRequestOnNextProbe is not null)
|
||||
{
|
||||
ReconciliationRequest = ReplacementRequestOnNextProbe;
|
||||
ReplacementRequestOnNextProbe = null;
|
||||
}
|
||||
return FirewallUnavailable
|
||||
? Task.FromException<bool>(new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode))
|
||||
: Task.FromResult(IsConnected(link));
|
||||
}
|
||||
|
||||
public Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ReconciliationStatusCalls++;
|
||||
return Task.FromResult(ReconciliationRequest);
|
||||
}
|
||||
|
||||
public Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
|
||||
{
|
||||
CompleteReconciliationCalls++;
|
||||
CompletedGenerations.Add(generation);
|
||||
if (ReconciliationRequest == generation)
|
||||
{
|
||||
ReconciliationRequest = null;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestTimeProvider(DateTimeOffset now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => now;
|
||||
public void Advance(TimeSpan value) => now += value;
|
||||
}
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ SOURCE_NODE_ID="${SOURCE_NODE_ID:-ai-agent}"
|
|||
HOME_NODE_ID="${HOME_NODE_ID:-home}"
|
||||
SECOND_NODE_ID="${SECOND_NODE_ID:-second}"
|
||||
TARGET_PORT="${TARGET_PORT:-22}"
|
||||
LINK_RECONCILIATION_SECONDS="${LINK_RECONCILIATION_SECONDS:-300}"
|
||||
CONTROL_DEVICE_ID="acceptance-$(date +%s)"
|
||||
INSTALLER_COMMAND="${INSTALLER_COMMAND:-sudo /usr/local/sbin/ochenstarik-server-monitor-manager.sh}"
|
||||
WORK_DIRECTORY="$(mktemp -d)"
|
||||
|
|
@ -48,30 +49,53 @@ source_ssh() {
|
|||
"$SOURCE_SSH_USER@$SOURCE_SSH_HOST" "$@"
|
||||
}
|
||||
|
||||
expect_reachable() {
|
||||
probe_reachable() {
|
||||
local ip="$1"
|
||||
source_ssh "nc -z -w 5 '$ip' '$TARGET_PORT'"
|
||||
}
|
||||
|
||||
expect_reachable() {
|
||||
local ip="$1"
|
||||
probe_reachable "$ip" || {
|
||||
echo "Expected access from $SOURCE_NODE_ID to $ip:$TARGET_PORT" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
probe_blocked() {
|
||||
local ip="$1"
|
||||
! probe_reachable "$ip"
|
||||
}
|
||||
|
||||
expect_blocked() {
|
||||
local ip="$1"
|
||||
if source_ssh "nc -z -w 5 '$ip' '$TARGET_PORT'"; then
|
||||
if ! probe_blocked "$ip"; then
|
||||
echo "Unexpected access from $SOURCE_NODE_ID to $ip:$TARGET_PORT" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_factual_status() {
|
||||
probe_factual_status() {
|
||||
local target="$1"
|
||||
local expected="$2"
|
||||
local command output
|
||||
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
||||
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
||||
output="$(hub_ssh "$command")"
|
||||
[[ "$output" == "$expected" ]] || {
|
||||
[[ "$output" == "$expected" ]]
|
||||
}
|
||||
|
||||
expect_factual_status() {
|
||||
local target="$1"
|
||||
local expected="$2"
|
||||
if ! probe_factual_status "$target" "$expected"; then
|
||||
local command output
|
||||
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
||||
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
||||
output="$(hub_ssh "$command")"
|
||||
echo "Unexpected factual Link status for $SOURCE_NODE_ID -> $target: $output (expected $expected)" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
decode_base64url() {
|
||||
|
|
@ -85,11 +109,11 @@ decode_base64url() {
|
|||
printf '%s' "$value" | base64 --decode
|
||||
}
|
||||
|
||||
echo '[1/11] Checking installed services on all three nodes'
|
||||
echo '[1/12] Checking installed services on all three nodes'
|
||||
hub_ssh "sudo systemctl is-active ochenstarik-smm-control.service >/dev/null"
|
||||
source_ssh "sudo systemctl is-active ochenstarik-smm-agent.service >/dev/null"
|
||||
|
||||
echo '[2/11] Creating an isolated operator identity for this acceptance run'
|
||||
echo '[2/12] Creating an isolated operator identity for this acceptance run'
|
||||
device_code="$(hub_ssh "$INSTALLER_COMMAND control-device-code '$CONTROL_DEVICE_ID'" | tr -d '\r' | grep -o 'SMMDEV1-[A-Za-z0-9_-]*' | tail -n1)"
|
||||
[[ -n "$device_code" ]] || { echo 'Hub did not return SMMDEV1 code' >&2; exit 1; }
|
||||
decode_base64url "${device_code#SMMDEV1-}" >"$WORK_DIRECTORY/device.env"
|
||||
|
|
@ -147,7 +171,7 @@ disable_link() {
|
|||
"$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" '{idempotencyKey:$key}')"
|
||||
}
|
||||
|
||||
echo '[3/11] Confirming all expected Agent identities are online'
|
||||
echo '[3/12] Confirming all expected Agent identities are online'
|
||||
agents="$(api_get '/api/v1/control/agents')"
|
||||
for node in "$SOURCE_NODE_ID" "$HOME_NODE_ID" "$SECOND_NODE_ID"; do
|
||||
jq -e --arg node "$node" '.[] | select(.nodeId == $node)' <<<"$agents" >/dev/null || {
|
||||
|
|
@ -156,7 +180,7 @@ for node in "$SOURCE_NODE_ID" "$HOME_NODE_ID" "$SECOND_NODE_ID"; do
|
|||
}
|
||||
done
|
||||
|
||||
echo '[4/11] Creating independent Links to home and second server'
|
||||
echo '[4/12] Creating independent Links to home and second server'
|
||||
home_link="$(create_link "$HOME_NODE_ID" 0)"
|
||||
second_link="$(create_link "$SECOND_NODE_ID" 0)"
|
||||
LINK_HOME_ID="$(jq -r '.id' <<<"$home_link")"
|
||||
|
|
@ -166,18 +190,18 @@ jq -e '.desiredState == "Active" and .actualState == "Active" and .lastError ==
|
|||
expect_factual_status "$HOME_NODE_ID" active
|
||||
expect_factual_status "$SECOND_NODE_ID" active
|
||||
|
||||
echo '[5/11] Verifying routed access through both Links'
|
||||
echo '[5/12] Verifying routed access through both Links'
|
||||
expect_reachable "$HOME_WG_IP"
|
||||
expect_reachable "$SECOND_WG_IP"
|
||||
|
||||
echo '[6/11] Disabling only the second Link'
|
||||
echo '[6/12] Disabling only the second Link'
|
||||
disable_link "$LINK_SECOND_ID" | jq -e \
|
||||
'.desiredState == "Disabled" and .actualState == "Disabled" and .lastError == null' >/dev/null
|
||||
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||
expect_reachable "$HOME_WG_IP"
|
||||
expect_blocked "$SECOND_WG_IP"
|
||||
|
||||
echo '[7/11] Verifying automatic TTL expiration'
|
||||
echo '[7/12] Verifying automatic TTL expiration'
|
||||
ttl_link="$(create_link "$SECOND_NODE_ID" 1)"
|
||||
ttl_id="$(jq -r '.id' <<<"$ttl_link")"
|
||||
expect_reachable "$SECOND_WG_IP"
|
||||
|
|
@ -196,13 +220,34 @@ expect_factual_status "$SECOND_NODE_ID" disabled
|
|||
expect_blocked "$SECOND_WG_IP"
|
||||
expect_reachable "$HOME_WG_IP"
|
||||
|
||||
echo '[8/11] Creating and validating a Control backup'
|
||||
if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
|
||||
echo '[8/12] Restoring the base firewall without restarting any Node'
|
||||
hub_ssh 'sudo /usr/local/sbin/ochenstarik-smm-emergency firewall-restore'
|
||||
deadline=$((SECONDS + LINK_RECONCILIATION_SECONDS + 30))
|
||||
while ((SECONDS < deadline)); do
|
||||
if probe_factual_status "$HOME_NODE_ID" active \
|
||||
&& probe_factual_status "$SECOND_NODE_ID" disabled \
|
||||
&& probe_reachable "$HOME_WG_IP" \
|
||||
&& probe_blocked "$SECOND_WG_IP"; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
expect_factual_status "$HOME_NODE_ID" active
|
||||
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||
expect_reachable "$HOME_WG_IP"
|
||||
expect_blocked "$SECOND_WG_IP"
|
||||
else
|
||||
echo '[8/12] Firewall restore reconciliation skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
|
||||
fi
|
||||
|
||||
echo '[9/12] Creating and validating a Control backup'
|
||||
backup_path="$(hub_ssh "sudo systemd-run --wait --pipe --quiet --collect --uid=ochenstarik-smm-control --gid=ochenstarik-smm-control -p EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-create" | grep '/backup-' | tail -n1)"
|
||||
[[ "$backup_path" == */backup-* ]] || { echo 'Backup command did not return a backup path' >&2; exit 1; }
|
||||
hub_ssh "sudo test -s '$backup_path/manifest.json' && sudo test -s '$backup_path/control.db' && sudo test -s '$backup_path/control-ca.pfx'"
|
||||
|
||||
if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
|
||||
echo '[9/11] Restoring the verified backup and restarting Control'
|
||||
echo '[10/12] Restoring the verified backup and restarting Control'
|
||||
hub_ssh "sudo sh -c 'set -a; . /etc/ochenstarik-server-monitor-manager/control.env; set +a; systemctl stop ochenstarik-smm-control.service; /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-restore \"\$1\"; status=\$?; systemctl start ochenstarik-smm-control.service; exit \$status' sh '$backup_path'"
|
||||
for _ in {1..30}; do
|
||||
if api_get '/healthz' >/dev/null 2>&1; then
|
||||
|
|
@ -216,11 +261,11 @@ if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
|
|||
expect_factual_status "$HOME_NODE_ID" active
|
||||
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||
else
|
||||
echo '[9/11] Restore check skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
|
||||
echo '[10/12] Restore check skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
|
||||
fi
|
||||
|
||||
if [[ "${SMM_ACCEPT_REBOOT:-0}" == '1' ]]; then
|
||||
echo '[10/11] Rebooting the Hub and source Node and rechecking policy state'
|
||||
echo '[11/12] Rebooting the Hub and source Node and rechecking policy state'
|
||||
hub_ssh 'sudo systemctl reboot' || true
|
||||
source_ssh 'sudo systemctl reboot' || true
|
||||
sleep 15
|
||||
|
|
@ -235,10 +280,10 @@ if [[ "${SMM_ACCEPT_REBOOT:-0}" == '1' ]]; then
|
|||
expect_factual_status "$HOME_NODE_ID" active
|
||||
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||
else
|
||||
echo '[10/11] Reboot check skipped; set SMM_ACCEPT_REBOOT=1 to enable it'
|
||||
echo '[11/12] Reboot check skipped; set SMM_ACCEPT_REBOOT=1 to enable it'
|
||||
fi
|
||||
|
||||
echo '[11/11] Revoking the temporary Operator certificate'
|
||||
echo '[12/12] Revoking the temporary Operator certificate'
|
||||
api_post "/api/v1/control/devices/$CONTROL_DEVICE_ID/reenroll" \
|
||||
"$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" \
|
||||
'{reason:"three-server acceptance completed",idempotencyKey:$key}')" \
|
||||
|
|
|
|||
|
|
@ -6,11 +6,13 @@ root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|||
bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
|
||||
helper="$root/deploy/ochenstarik-smm-policy-apply"
|
||||
emergency="$root/deploy/ochenstarik-smm-emergency"
|
||||
acceptance="$root/tests/acceptance/three-server-mesh.sh"
|
||||
|
||||
grep -Fq 'if ! listing="$(/usr/sbin/nft -a list chain' "$helper" || {
|
||||
grep -Fq 'listing="$(/usr/sbin/nft -a list chain' "$helper" || {
|
||||
printf '%s\n' "policy status probe must fail closed when nftables cannot be inspected" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -Fq "grep -Eiq 'No such file or directory|does not exist'" "$helper"
|
||||
provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service"
|
||||
|
||||
grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit"
|
||||
|
|
@ -61,6 +63,13 @@ grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$vers
|
|||
emergency_help="$(bash "$emergency" --help)"
|
||||
grep -Fq 'mesh-disable' <<<"$emergency_help"
|
||||
grep -Fq 'firewall-restore' <<<"$emergency_help"
|
||||
grep -Fq 'readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"' "$emergency"
|
||||
grep -Fq 'chown root:root "$temporary_marker"' "$emergency"
|
||||
grep -Fq 'chmod 0600 "$temporary_marker"' "$emergency"
|
||||
grep -Fq 'mv -f -- "$temporary_marker" "$RECONCILE_MARKER"' "$emergency"
|
||||
grep -Fq '/usr/bin/flock -x 9' "$emergency"
|
||||
grep -Fq 'generation="$(</proc/sys/kernel/random/uuid)"' "$emergency"
|
||||
[[ "$(grep -Fc ' request_reconciliation' "$emergency")" -ge 2 ]]
|
||||
|
||||
if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then
|
||||
printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2
|
||||
|
|
@ -97,8 +106,83 @@ if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
|||
printf '%s\n' "policy helper unexpectedly accepted extra link-status arguments" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
|
||||
reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)"
|
||||
printf '%s\n' "$generation_a" >"$reconcile_marker"
|
||||
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||
bash "$helper" reconcile-status)" == "requested:$generation_a" ]]
|
||||
printf '%s\n' "$generation_b" >"$reconcile_marker"
|
||||
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||
bash "$helper" reconcile-complete "$generation_a" >/dev/null
|
||||
[[ "$(<"$reconcile_marker")" == "$generation_b" ]]
|
||||
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||
bash "$helper" reconcile-complete "$generation_b" >/dev/null
|
||||
[[ ! -e "$reconcile_marker" ]]
|
||||
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||
bash "$helper" reconcile-status)" == 'complete' ]]
|
||||
if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||
bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then
|
||||
printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
firewall_error="$(mktemp -t smm-firewall-error.XXXXXXXX)"
|
||||
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
||||
SMM_POLICY_FIREWALL_UNAVAILABLE=1 bash "$helper" \
|
||||
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
|
||||
printf '%s\n' "missing firewall unexpectedly produced a factual Link status" >&2
|
||||
exit 1
|
||||
else
|
||||
[[ $? -eq 79 ]]
|
||||
fi
|
||||
[[ "$(<"$firewall_error")" == 'mesh.firewall-unavailable' ]]
|
||||
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
||||
SMM_POLICY_FIREWALL_ERROR='permission denied' bash "$helper" \
|
||||
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
|
||||
printf '%s\n' "unknown nft inspection error unexpectedly produced a factual Link status" >&2
|
||||
exit 1
|
||||
else
|
||||
[[ $? -eq 78 ]]
|
||||
fi
|
||||
grep -Fq 'permission denied' "$firewall_error"
|
||||
if grep -Fq 'mesh.firewall-unavailable' "$firewall_error"; then
|
||||
printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -f -- "$firewall_error" "$reconcile_marker"
|
||||
rm -f -- "$policy_state"
|
||||
|
||||
extract_shell_function() {
|
||||
local name="$1"
|
||||
awk -v signature="$name() {" '
|
||||
$0 == signature { emitting = 1 }
|
||||
emitting { print }
|
||||
emitting && $0 == "}" { exit }
|
||||
' "$acceptance"
|
||||
}
|
||||
eval "$(extract_shell_function probe_factual_status)"
|
||||
SOURCE_NODE_ID=source
|
||||
TARGET_PORT=22
|
||||
probe_counter="$(mktemp -t smm-factual-probe.XXXXXXXX)"
|
||||
printf '%s\n' 0 >"$probe_counter"
|
||||
hub_ssh() {
|
||||
local count
|
||||
count="$(( $(<"$probe_counter") + 1 ))"
|
||||
printf '%s\n' "$count" >"$probe_counter"
|
||||
[[ "$count" -eq 1 ]] && printf '%s\n' disabled || printf '%s\n' active
|
||||
}
|
||||
if probe_factual_status target active; then
|
||||
printf '%s\n' "initial factual mismatch unexpectedly passed" >&2
|
||||
exit 1
|
||||
fi
|
||||
probe_factual_status target active || {
|
||||
printf '%s\n' "factual probe did not allow convergence after an initial mismatch" >&2
|
||||
exit 1
|
||||
}
|
||||
rm -f -- "$probe_counter" "${reconcile_marker}.lock"
|
||||
|
||||
fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)"
|
||||
trap 'rm -rf -- "$fixture"' EXIT
|
||||
mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ $linksCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
|
|||
Join-Path $root 'src\ServerMonitorManager.Desktop\Pages\LinksPage.xaml.cs')
|
||||
$mainCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
|
||||
Join-Path $root 'src\ServerMonitorManager.Desktop\MainPage.xaml.cs')
|
||||
$meshModelsCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
|
||||
Join-Path $root 'src\ServerMonitorManager.Desktop\MeshModels.cs')
|
||||
$appCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
|
||||
Join-Path $root 'src\ServerMonitorManager.Desktop\App.xaml.cs')
|
||||
$sshCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
|
||||
|
|
@ -52,6 +54,22 @@ if ($linksCode.IndexOf(
|
|||
'LinksList.SelectedItem as MeshLinkViewModel', [StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Links page must pass its selected Link to the command handler.'
|
||||
}
|
||||
if ($linksXaml.IndexOf('x:Name="FirewallUnavailableInfo"', [StringComparison]::Ordinal) -lt 0 -or
|
||||
$linksXaml.IndexOf('Message="Mesh firewall ', [StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Links page must expose one persistent Mesh firewall unavailable banner.'
|
||||
}
|
||||
if ($linksCode.IndexOf('SetFirewallUnavailable', [StringComparison]::Ordinal) -lt 0 -or
|
||||
$mainCode.IndexOf('MeshLinkViewModel.FirewallUnavailableErrorCode', [StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Desktop must project both event and persisted Mesh firewall unavailable state.'
|
||||
}
|
||||
if ($meshModelsCode.IndexOf('LastError == FirewallUnavailableErrorCode ? string.Empty',
|
||||
[StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Shared Mesh firewall errors must be suppressed from individual Link rows.'
|
||||
}
|
||||
if ($mainCode.IndexOf('Servers.Count > 0 || _control.IsConfigured', [StringComparison]::Ordinal) -lt 0 -or
|
||||
$mainCode.IndexOf('await RefreshControlMeshAsync(showSuccess: false);', [StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Configured Control state must refresh even when no SSH profiles exist.'
|
||||
}
|
||||
if ($mainCode.IndexOf(
|
||||
'MeshLinksList.SelectedItem = selectedLink;', [StringComparison]::Ordinal) -lt 0) {
|
||||
throw 'Main page must synchronize the selected Link before disconnecting it.'
|
||||
|
|
|
|||
Loading…
Reference in a new issue