ochenstarik/server-monitor-manager
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases 17 Packages Wiki Activity Actions 2

feat(control): add background Link reconciliation #12

Merged
ochenstarik-ui merged 1 commit from hermes/task3-b2-background-reconciliation into main 2026-08-03 06:55:49 +00:00
Conversation 0 Commits 1 Files changed 21 +1116 -175
21 changed files with 1116 additions and 175 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
deploy/ochenstarik-server-monitor-manager.sh
View file

@ -494,6 +494,7 @@ Control__CertificateAuthorityPath=$ETC_DIR/control-ca.pfx
Control__BackupDirectory=$STATE_DIR/backups Control__BackupDirectory=$STATE_DIR/backups
Control__HubHelperPath=$POLICY_HELPER Control__HubHelperPath=$POLICY_HELPER
Control__PrivilegeEscalationPath=/usr/bin/sudo Control__PrivilegeEscalationPath=/usr/bin/sudo
Control__LinkReconciliationSeconds=300
EOF EOF
printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url" printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url"
chown root:"$CONTROL_USER" "$ETC_DIR/control.env" chown root:"$CONTROL_USER" "$ETC_DIR/control.env"

24
deploy/ochenstarik-smm-emergency
View file

@ -1,11 +1,14 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -Eeuo pipefail set -Eeuo pipefail
IFS=$'\n\t' IFS=$'\n\t'
umask 077
readonly PROGRAM="ochenstarik-smm-emergency" readonly PROGRAM="ochenstarik-smm-emergency"
readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager" readonly STATE_DIR="/var/lib/ochenstarik-server-monitor-manager"
readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager" readonly ETC_DIR="/etc/ochenstarik-server-monitor-manager"
readonly MARKER="$STATE_DIR/mesh/emergency-disabled" readonly MARKER="$STATE_DIR/mesh/emergency-disabled"
readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"
readonly RECONCILE_LOCK="$STATE_DIR/mesh/reconcile-requested.lock"
readonly CONTROL_UNIT="ochenstarik-smm-control.service" readonly CONTROL_UNIT="ochenstarik-smm-control.service"
readonly AGENT_UNIT="ochenstarik-smm-agent.service" readonly AGENT_UNIT="ochenstarik-smm-agent.service"
readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service" readonly FIREWALL_UNIT="ochenstarik-smm-firewall.service"
@ -71,6 +74,25 @@ delete_project_firewall() {
fi fi
} }
request_reconciliation() {
local marker_directory temporary_marker generation
marker_directory="$(dirname "$RECONCILE_MARKER")"
install -d -o root -g root -m 0700 "$marker_directory"
exec 9>"$RECONCILE_LOCK"
chown root:root "$RECONCILE_LOCK"
chmod 0600 "$RECONCILE_LOCK"
/usr/bin/flock -x 9
generation="$(</proc/sys/kernel/random/uuid)"
temporary_marker="$(mktemp "$marker_directory/.reconcile-requested.XXXXXXXX")"
if ! printf '%s\n' "$generation" >"$temporary_marker" \
|| ! chown root:root "$temporary_marker" \
|| ! chmod 0600 "$temporary_marker" \
|| ! mv -f -- "$temporary_marker" "$RECONCILE_MARKER"; then
rm -f -- "$temporary_marker"
fail "could not create the Link reconciliation request"
fi
}
mesh_disable() { mesh_disable() {
require_root require_root
systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true systemctl disable --now "$WIREGUARD_UNIT" 2>/dev/null || true
@ -97,6 +119,7 @@ restore_project_firewall() {
if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then if systemctl list-unit-files "$FIREWALL_UNIT" --no-legend 2>/dev/null | grep -q "^$FIREWALL_UNIT"; then
systemctl enable "$FIREWALL_UNIT" >/dev/null systemctl enable "$FIREWALL_UNIT" >/dev/null
fi fi
request_reconciliation
log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links." log "Base deny-by-default Mesh firewall restored; Control must reconcile active Links."
} }
@ -108,6 +131,7 @@ mesh_enable() {
fi fi
systemctl enable "$WIREGUARD_UNIT" >/dev/null systemctl enable "$WIREGUARD_UNIT" >/dev/null
systemctl restart "$WIREGUARD_UNIT" systemctl restart "$WIREGUARD_UNIT"
request_reconciliation
rm -f -- "$MARKER" rm -f -- "$MARKER"
log "Mesh enabled locally." log "Mesh enabled locally."
} }

86
deploy/ochenstarik-smm-policy-apply
View file

@ -1,23 +1,33 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -Eeuo pipefail set -Eeuo pipefail
IFS=$'\n\t' IFS=$'\n\t'
export LC_ALL=C
umask 077
readonly TABLE_FAMILY="inet" readonly TABLE_FAMILY="inet"
readonly TABLE_NAME="ochenstarik_smm" readonly TABLE_NAME="ochenstarik_smm"
readonly CHAIN_NAME="links" readonly CHAIN_NAME="links"
fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; } fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; }
firewall_unavailable() { printf '%s\n' "mesh.firewall-unavailable" >&2; exit 79; }
testing="${SMM_POLICY_TESTING:-0}" testing="${SMM_POLICY_TESTING:-0}"
if [[ "$testing" != "1" ]]; then if [[ "$testing" != "1" ]]; then
[[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required" [[ ${EUID:-$(id -u)} -eq 0 ]] || fail "root is required"
readonly STATE_FILE="/var/lib/ochenstarik-server-monitor-manager/mesh/nodes.tsv" readonly STATE_FILE="/var/lib/ochenstarik-server-monitor-manager/mesh/nodes.tsv"
readonly RECONCILE_MARKER="/var/lib/ochenstarik-server-monitor-manager/mesh/reconcile-requested"
readonly RECONCILE_LOCK="/var/lib/ochenstarik-server-monitor-manager/mesh/reconcile-requested.lock"
readonly FLOCK_COMMAND="/usr/bin/flock"
else else
readonly STATE_FILE="${SMM_POLICY_STATE_FILE:?SMM_POLICY_STATE_FILE is required in testing mode}" readonly STATE_FILE="${SMM_POLICY_STATE_FILE:-/dev/null}"
readonly RECONCILE_MARKER="${SMM_POLICY_RECONCILE_MARKER:-${TMPDIR:-/tmp}/smm-policy-reconcile-requested}"
readonly RECONCILE_LOCK="${SMM_POLICY_RECONCILE_LOCK:-${RECONCILE_MARKER}.lock}"
readonly FLOCK_COMMAND="${SMM_POLICY_FLOCK:-/usr/bin/flock}"
fi fi
node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$'
ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$' ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
generation_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
validate_node_id() { validate_node_id() {
[[ "$1" =~ $node_pattern ]] || fail "invalid node id" [[ "$1" =~ $node_pattern ]] || fail "invalid node id"
@ -59,20 +69,38 @@ run_nft() {
fi fi
} }
inspect_firewall() {
local listing
if [[ "$testing" == "1" ]]; then
[[ "${SMM_POLICY_FIREWALL_UNAVAILABLE:-0}" != "1" ]] || firewall_unavailable
[[ -z "${SMM_POLICY_FIREWALL_ERROR:-}" ]] \
|| fail "could not inspect nftables Link policy: $SMM_POLICY_FIREWALL_ERROR"
return 0
fi
if listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" 2>&1)"; then
printf '%s\n' "$listing"
return 0
fi
if grep -Eiq 'No such file or directory|does not exist' <<<"$listing"; then
firewall_unavailable
fi
fail "could not inspect nftables Link policy: ${listing%%$'\n'*}"
}
ensure_firewall_available() {
inspect_firewall >/dev/null
}
rule_exists() { rule_exists() {
local comment="$1" listing local comment="$1" listing
if [[ "$testing" == "1" ]]; then listing="$(inspect_firewall)"
return 1
fi
if ! listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME")"; then
fail "could not inspect nftables Link policy"
fi
grep -Fq "comment \"$comment\"" <<<"$listing" grep -Fq "comment \"$comment\"" <<<"$listing"
} }
connect_rule() { connect_rule() {
local source_id="$1" target_id="$2" protocol="$3" port="$4" local source_id="$1" target_id="$2" protocol="$3" port="$4"
local source_ip target_ip comment local source_ip target_ip comment
ensure_firewall_available
source_ip="$(lookup_node_ip "$source_id")" source_ip="$(lookup_node_ip "$source_id")"
target_ip="$(lookup_node_ip "$target_id")" target_ip="$(lookup_node_ip "$target_id")"
comment="smm:${source_id}:${target_id}:${protocol}:${port}" comment="smm:${source_id}:${target_id}:${protocol}:${port}"
@ -84,6 +112,7 @@ connect_rule() {
disconnect_rule() { disconnect_rule() {
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle
ensure_firewall_available
# Resolve both identities before touching firewall state. # Resolve both identities before touching firewall state.
lookup_node_ip "$source_id" >/dev/null lookup_node_ip "$source_id" >/dev/null
lookup_node_ip "$target_id" >/dev/null lookup_node_ip "$target_id" >/dev/null
@ -104,6 +133,7 @@ disconnect_rule() {
status_rule() { status_rule() {
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment local source_id="$1" target_id="$2" protocol="$3" port="$4" comment
ensure_firewall_available
# A status is meaningful only for identities currently known to the mesh. # A status is meaningful only for identities currently known to the mesh.
lookup_node_ip "$source_id" >/dev/null lookup_node_ip "$source_id" >/dev/null
lookup_node_ip "$target_id" >/dev/null lookup_node_ip "$target_id" >/dev/null
@ -115,7 +145,49 @@ status_rule() {
fi fi
} }
reconcile_status() {
local generation
exec 9>"$RECONCILE_LOCK"
chmod 0600 "$RECONCILE_LOCK"
"$FLOCK_COMMAND" -x 9
if [[ ! -f "$RECONCILE_MARKER" ]]; then
printf '%s\n' complete
return
fi
IFS= read -r generation <"$RECONCILE_MARKER" || fail "invalid reconciliation marker"
[[ "$generation" =~ $generation_pattern ]] || fail "invalid reconciliation marker"
printf 'requested:%s\n' "$generation"
}
reconcile_complete() {
local expected_generation="$1" current_generation=''
[[ "$expected_generation" =~ $generation_pattern ]] || fail "invalid reconciliation generation"
exec 9>"$RECONCILE_LOCK"
chmod 0600 "$RECONCILE_LOCK"
"$FLOCK_COMMAND" -x 9
if [[ -f "$RECONCILE_MARKER" ]]; then
IFS= read -r current_generation <"$RECONCILE_MARKER" || fail "invalid reconciliation marker"
[[ "$current_generation" =~ $generation_pattern ]] || fail "invalid reconciliation marker"
if [[ "$current_generation" == "$expected_generation" ]]; then
rm -f -- "$RECONCILE_MARKER"
fi
fi
printf '%s\n' complete
}
action="${1:-}" action="${1:-}"
case "$action" in
reconcile-status)
[[ $# -eq 1 ]] || fail "invalid reconcile-status argument count"
reconcile_status
exit 0
;;
reconcile-complete)
[[ $# -eq 2 ]] || fail "invalid reconcile-complete argument count"
reconcile_complete "$2"
exit 0
;;
esac
validate_rule "$@" validate_rule "$@"
case "$action" in case "$action" in
link-connect) connect_rule "$2" "$3" "$4" "$5" ;; link-connect) connect_rule "$2" "$3" "$4" "$5" ;;

2
docs/linux-bootstrap.md
View file

@ -108,4 +108,4 @@ sudo ochenstarik-smm-emergency firewall-restore
sudo ochenstarik-smm-emergency mesh-enable sudo ochenstarik-smm-emergency mesh-enable
``` ```
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default; разрешающие Link-правила после этого должен повторно применить Control. Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` запускайте только после проверки конфигурации и доступности Hub. `mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default и атомарно создаёт root-only запрос реконсиляции с уникальным generation. Control немедленно выполняет первый фоновый проход после старта, затем сверяет все действующие Link-политики не реже настроенного интервала `Control__LinkReconciliationSeconds` (по умолчанию 300 секунд). Новый marker запускает внеочередной проход на следующем poll tick, но не обходит backoff недоступного firewall. Helper удаляет только тот generation, который был прочитан перед успешно завершившимся проходом; более новый запрос сохраняется. При недоступной таблице запрос остаётся для retry с backoff, а Desktop показывает единый баннер «Mesh firewall не загружен». Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` также создаёт запрос реконсиляции; запускайте его только после проверки конфигурации и доступности Hub.

2
docs/roadmap.md
View file

@ -57,6 +57,8 @@
- [x] обязательное восстановление disabled policy после reconnect; - [x] обязательное восстановление disabled policy после reconnect;
- [x] append-only audit операций Link; - [x] append-only audit операций Link;
- [x] интеграционные тесты kill switch, process restart и helper failure; - [x] интеграционные тесты kill switch, process restart и helper failure;
- [x] B-2: независимая фоновая и emergency-triggered реконсиляция, агрегированное состояние недоступного Mesh firewall и Desktop banner;
- [ ] B-3: развести результат реконсиляции на `Examined` / `Converged` / `Failed` (M2), выбрать фильтр или retention завершённых Disabled-политик (M4) и типизировать ещё не активированный Mesh Node (M5). Эти изменения вынесены отдельно, потому что меняют API/UI semantics и helper classification, тогда как B-2 ограничен восстановлением фактических правил и общим firewall failure;
- [ ] выполнить физический acceptance Hub + source Node + два destination Node с WireGuard/nftables/reboot. - [ ] выполнить физический acceptance Hub + source Node + два destination Node с WireGuard/nftables/reboot.
## Этап 4 — мониторинг и терминал ## Этап 4 — мониторинг и терминал

2
src/ServerMonitorManager.Control/ControlOptions.cs
View file

@ -24,6 +24,8 @@ public sealed class ControlOptions
public int LinkExpirationPollSeconds { get; init; } = 15; public int LinkExpirationPollSeconds { get; init; } = 15;
public int LinkReconciliationSeconds { get; init; } = 300;
public string BackupDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/backups"; public string BackupDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/backups";
public int BackupIntervalHours { get; init; } = 24; public int BackupIntervalHours { get; init; } = 24;

62
src/ServerMonitorManager.Control/ControlStore.cs
View file

@ -1353,6 +1353,17 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
return link; return link;
} }
public async Task<LinkPolicy?> GetLinkAsync(
string id,
CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken);
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
var link = await ReadLinkAsync(connection, transaction, id, cancellationToken);
await transaction.CommitAsync(cancellationToken);
return link;
}
public async Task<IReadOnlyList<LinkPolicy>> ListLinksAsync(CancellationToken cancellationToken = default) public async Task<IReadOnlyList<LinkPolicy>> ListLinksAsync(CancellationToken cancellationToken = default)
{ {
var result = new List<LinkPolicy>(); var result = new List<LinkPolicy>();
@ -1367,6 +1378,57 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
return result; return result;
} }
public async Task<IReadOnlyList<LinkPolicy>> ListEffectiveLinksAsync(
CancellationToken cancellationToken = default)
{
var result = new List<LinkPolicy>();
await using var connection = await OpenAsync(cancellationToken);
var command = connection.CreateCommand();
command.CommandText = """
SELECT current.*
FROM links AS current
WHERE (current.desired_state = 'Active'
OR (current.desired_state = 'Disabled' AND current.actual_state != 'Disabled'))
AND NOT EXISTS (
SELECT 1 FROM links AS newer
WHERE newer.source_node_id = current.source_node_id
AND newer.target_node_id = current.target_node_id
AND newer.protocol = current.protocol
AND newer.port = current.port
AND newer.version > current.version)
ORDER BY current.version;
""";
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
result.Add(ReadLink(reader));
}
return result;
}
public async Task<bool> IsEffectiveLinkAsync(
LinkPolicy link,
CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken);
var command = connection.CreateCommand();
command.CommandText = """
SELECT NOT EXISTS(
SELECT 1 FROM links
WHERE source_node_id = $source
AND target_node_id = $target
AND protocol = $protocol
AND port = $port
AND version > $version);
""";
command.Parameters.AddWithValue("$source", link.SourceNodeId);
command.Parameters.AddWithValue("$target", link.TargetNodeId);
command.Parameters.AddWithValue("$protocol", link.Protocol);
command.Parameters.AddWithValue("$port", link.Port);
command.Parameters.AddWithValue("$version", link.Version);
return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)) == 1;
}
public async Task<IReadOnlyList<LinkPolicy>> ListExpiredLinksAsync( public async Task<IReadOnlyList<LinkPolicy>> ListExpiredLinksAsync(
DateTimeOffset now, DateTimeOffset now,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)

34
src/ServerMonitorManager.Control/LinkPolicyApplier.cs
View file

@ -9,6 +9,17 @@ public interface ILinkPolicyApplier
Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken); Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken);
Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken); Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken);
Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken); Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken);
Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
=> Task.FromResult<string?>(null);
Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
=> Task.CompletedTask;
}
public sealed class MeshFirewallUnavailableException : InvalidOperationException
{
public MeshFirewallUnavailableException(string message) : base(message)
{
}
} }
public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkPolicyApplier public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkPolicyApplier
@ -55,6 +66,24 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
}; };
} }
public async Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
=> await RunAsync(["reconcile-status"], cancellationToken) switch
{
"complete" => null,
var status when status.StartsWith("requested:", StringComparison.Ordinal)
&& Guid.TryParseExact(status[10..], "D", out _) => status[10..],
_ => throw new InvalidOperationException("Hub policy helper returned an invalid reconciliation marker status.")
};
public async Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
{
if (!Guid.TryParseExact(generation, "D", out _))
{
throw new ArgumentException("Invalid reconciliation generation.", nameof(generation));
}
_ = await RunAsync(["reconcile-complete", generation], cancellationToken);
}
private async Task<string> RunAsync(IReadOnlyList<string> arguments, CancellationToken cancellationToken) private async Task<string> RunAsync(IReadOnlyList<string> arguments, CancellationToken cancellationToken)
{ {
var startInfo = new ProcessStartInfo var startInfo = new ProcessStartInfo
@ -80,6 +109,11 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
if (process.ExitCode != 0) if (process.ExitCode != 0)
{ {
var message = (await error).Trim(); var message = (await error).Trim();
if (process.ExitCode == 79
&& string.Equals(message, LinkService.FirewallUnavailableCode, StringComparison.Ordinal))
{
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
}
throw new InvalidOperationException(string.IsNullOrWhiteSpace(message) throw new InvalidOperationException(string.IsNullOrWhiteSpace(message)
? $"Hub policy helper exited with code {process.ExitCode}." ? $"Hub policy helper exited with code {process.ExitCode}."
: message); : message);

96
src/ServerMonitorManager.Control/LinkReconciliationBackgroundService.cs Normal file
View file

@ -0,0 +1,96 @@
using Microsoft.Extensions.Options;
namespace ServerMonitorManager.Control;
public sealed class LinkReconciliationBackgroundService(
LinkService links,
ILinkPolicyApplier applier,
IOptions<ControlOptions> options,
TimeProvider timeProvider,
ILogger<LinkReconciliationBackgroundService> logger) : BackgroundService
{
private readonly SemaphoreSlim _passGate = new(1, 1);
private DateTimeOffset? _nextRegularAt;
private DateTimeOffset? _backoffUntil;
private int _unavailableAttempts;
internal async Task<LinkFullReconciliationResult?> RunOnceAsync(
CancellationToken cancellationToken = default)
{
await _passGate.WaitAsync(cancellationToken);
try
{
var now = timeProvider.GetUtcNow();
string? requestGeneration = null;
try
{
requestGeneration = await applier.GetReconciliationRequestAsync(cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(exception, "Could not inspect the Link reconciliation marker.");
}
if (_backoffUntil is not null && now < _backoffUntil)
{
return null;
}
if (requestGeneration is null && _nextRegularAt is not null && now < _nextRegularAt)
{
return null;
}
var result = await links.ReconcileAllAsync(cancellationToken);
var interval = TimeSpan.FromSeconds(options.Value.LinkReconciliationSeconds);
if (result.FirewallUnavailable)
{
_unavailableAttempts = Math.Min(_unavailableAttempts + 1, 4);
_backoffUntil = now + TimeSpan.FromTicks(interval.Ticks * _unavailableAttempts);
}
else
{
_unavailableAttempts = 0;
_backoffUntil = null;
_nextRegularAt = now + interval;
if (requestGeneration is not null && result.Failed == 0)
{
await applier.CompleteReconciliationAsync(requestGeneration, cancellationToken);
}
}
return result;
}
finally
{
_passGate.Release();
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var pollSeconds = Math.Min(options.Value.LinkReconciliationSeconds, 30);
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(pollSeconds), timeProvider);
do
{
try
{
var result = await RunOnceAsync(stoppingToken);
if (result is not null)
{
logger.LogInformation(
"Link reconciliation completed: {Examined} examined, {Failed} failed, firewall unavailable: {Unavailable}.",
result.Examined,
result.Failed,
result.FirewallUnavailable);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception exception)
{
logger.LogError(exception, "Link reconciliation failed.");
}
}
while (await timer.WaitForNextTickAsync(stoppingToken));
}
}

346
src/ServerMonitorManager.Control/LinkService.cs
View file

@ -9,6 +9,8 @@ public sealed class LinkService(
ILinkPolicyApplier applier, ILinkPolicyApplier applier,
ControlEventBroker events) ControlEventBroker events)
{ {
public const string FirewallUnavailableCode = "mesh.firewall-unavailable";
public const string FirewallAvailableCode = "mesh.firewall-available";
private readonly ConcurrentDictionary<string, SemaphoreSlim> _reconciliationLocks = new(); private readonly ConcurrentDictionary<string, SemaphoreSlim> _reconciliationLocks = new();
private readonly ConcurrentDictionary<string, SemaphoreSlim> _nodeLocks = new(); private readonly ConcurrentDictionary<string, SemaphoreSlim> _nodeLocks = new();
@ -25,35 +27,20 @@ public sealed class LinkService(
{ {
return link; return link;
} }
var gate = _reconciliationLocks.GetOrAdd(link.Id, static _ => new SemaphoreSlim(1, 1)); var gate = await AcquireLinkGateAsync(link.Id, cancellationToken);
await gate.WaitAsync(cancellationToken);
try try
{ {
var current = (await store.ListLinksAsync(cancellationToken)) var current = await store.GetLinkAsync(link.Id, cancellationToken)
.SingleOrDefault(candidate => candidate.Id == link.Id)
?? throw new InvalidOperationException("The persisted Link disappeared."); ?? throw new InvalidOperationException("The persisted Link disappeared.");
if (current.DesiredState != "Active") if (current.DesiredState != "Active" || !await store.IsEffectiveLinkAsync(current, cancellationToken))
{ {
return current; return current;
} }
link = current; if (!mutation.IsReplay)
Publish("link.connecting", link);
try
{ {
await applier.ApplyConnectAsync(link, cancellationToken); Publish("link.connecting", current);
await VerifyFactualStateAsync(link, expectedConnected: true, cancellationToken);
link = await store.SetLinkActualStateAsync(link.Id, "Active", null, actor, cancellationToken)
?? throw new InvalidOperationException("The persisted Link disappeared.");
Publish("link.active", link);
} }
catch (Exception exception) when (exception is not OperationCanceledException) return await ConvergeAsync(current, expectedConnected: true, actor, cancellationToken);
{
link = await store.SetLinkActualStateAsync(
link.Id, "Failed", CompactError(exception), actor, cancellationToken)
?? link;
Publish("link.failed", link);
}
return link;
} }
finally finally
{ {
@ -67,8 +54,7 @@ public sealed class LinkService(
string actor, string actor,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var gate = _reconciliationLocks.GetOrAdd(id, static _ => new SemaphoreSlim(1, 1)); var gate = await AcquireLinkGateAsync(id, cancellationToken);
await gate.WaitAsync(cancellationToken);
try try
{ {
return await DisableCoreAsync(id, request, actor, cancellationToken); return await DisableCoreAsync(id, request, actor, cancellationToken);
@ -99,7 +85,7 @@ public sealed class LinkService(
{ {
Publish("link.disconnecting", link); Publish("link.disconnecting", link);
} }
return await ConvergeDisabledCoreAsync(link, actor, cancellationToken); return await ConvergeAsync(link, expectedConnected: false, actor, cancellationToken);
} }
internal async Task<LinkPolicy> ConvergeDisabledAsync( internal async Task<LinkPolicy> ConvergeDisabledAsync(
@ -107,11 +93,10 @@ public sealed class LinkService(
string actor, string actor,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var gate = _reconciliationLocks.GetOrAdd(link.Id, static _ => new SemaphoreSlim(1, 1)); var gate = await AcquireLinkGateAsync(link.Id, cancellationToken);
await gate.WaitAsync(cancellationToken);
try try
{ {
return await ConvergeDisabledCoreAsync(link, actor, cancellationToken); return await ConvergeAsync(link, expectedConnected: false, actor, cancellationToken);
} }
finally finally
{ {
@ -130,54 +115,24 @@ public sealed class LinkService(
{ {
using var nodeLease = await AcquireNodeLocksAsync( using var nodeLease = await AcquireNodeLocksAsync(
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken); [candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
var gate = _reconciliationLocks.GetOrAdd(candidate.Id, static _ => new SemaphoreSlim(1, 1)); var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
await gate.WaitAsync(cancellationToken);
try try
{ {
var current = (await store.ListEffectiveLinksForNodeAsync(nodeId, cancellationToken)) var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
.SingleOrDefault(link => link.Id == candidate.Id); if (current is null || (current.SourceNodeId != nodeId && current.TargetNodeId != nodeId))
if (current is null)
{ {
continue; continue;
} }
var result = await ConvergeAsync(
var actor = $"system:reconnect:{nodeId}"; current,
var expectedConnected = current.DesiredState == "Active"; current.DesiredState == "Active",
var pendingState = expectedConnected ? "Connecting" : "Disconnecting"; $"system:reconnect:{nodeId}",
var completedState = expectedConnected ? "Active" : "Disabled"; cancellationToken);
var completedEvent = expectedConnected ? "link.active" : "link.disabled"; reconciled++;
var failureState = expectedConnected ? "Failed" : "Partial"; if (result.ActualState is "Failed" or "Partial")
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
var link = await store.SetLinkActualStateAsync(
current.Id, pendingState, null, actor, cancellationToken) ?? current;
Publish("link.reconciling", link);
try
{ {
var isConnected = await applier.IsConnectedAsync(link, cancellationToken);
if (isConnected != expectedConnected)
{
if (expectedConnected)
{
await applier.ApplyConnectAsync(link, cancellationToken);
}
else
{
await applier.ApplyDisconnectAsync(link, cancellationToken);
}
await VerifyFactualStateAsync(link, expectedConnected, cancellationToken);
}
link = await store.SetLinkActualStateAsync(
link.Id, completedState, null, actor, cancellationToken) ?? link;
Publish(completedEvent, link);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
link = await store.SetLinkActualStateAsync(
link.Id, failureState, CompactError(exception), actor, cancellationToken) ?? link;
Publish(failureEvent, link);
failed++; failed++;
} }
reconciled++;
} }
finally finally
{ {
@ -187,6 +142,69 @@ public sealed class LinkService(
return new LinkReconciliationResult(reconciled, failed); return new LinkReconciliationResult(reconciled, failed);
} }
public async Task<LinkFullReconciliationResult> ReconcileAllAsync(
CancellationToken cancellationToken = default)
{
var candidates = await store.ListEffectiveLinksAsync(cancellationToken);
var recoveringFirewall = candidates.Any(candidate =>
string.Equals(candidate.LastError, FirewallUnavailableCode, StringComparison.Ordinal));
var examined = 0;
var failed = 0;
var firewallUnavailable = false;
foreach (var candidate in candidates)
{
using var nodeLease = await AcquireNodeLocksAsync(
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
try
{
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
if (current is null || !IsEligibleForFullReconciliation(current))
{
continue;
}
var result = await ConvergeAsync(
current, current.DesiredState == "Active", "system:reconcile", cancellationToken);
examined++;
if (result.LastError == FirewallUnavailableCode)
{
firewallUnavailable = true;
}
else if (result.ActualState is "Failed" or "Partial")
{
failed++;
}
}
finally
{
gate.Release();
}
if (firewallUnavailable)
{
break;
}
}
if (firewallUnavailable)
{
await MarkFirewallUnavailableAsync(candidates, cancellationToken);
events.Publish(
FirewallUnavailableCode,
"mesh",
JsonSerializer.Serialize(
new ControlError(FirewallUnavailableCode), SmmJsonContext.Default.ControlError));
return new LinkFullReconciliationResult(examined, candidates.Count, true);
}
if (recoveringFirewall)
{
events.Publish(
FirewallAvailableCode,
"mesh",
JsonSerializer.Serialize(
new ControlError(FirewallAvailableCode), SmmJsonContext.Default.ControlError));
}
return new LinkFullReconciliationResult(examined, failed, false);
}
public async Task<LinkExpirationResult> ExpireDueLinksAsync( public async Task<LinkExpirationResult> ExpireDueLinksAsync(
DateTimeOffset now, DateTimeOffset now,
CancellationToken cancellationToken) CancellationToken cancellationToken)
@ -196,24 +214,30 @@ public sealed class LinkService(
var links = await store.ListExpiredLinksAsync(now, cancellationToken); var links = await store.ListExpiredLinksAsync(now, cancellationToken);
foreach (var candidate in links) foreach (var candidate in links)
{ {
var gate = _reconciliationLocks.GetOrAdd(candidate.Id, static _ => new SemaphoreSlim(1, 1)); var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
await gate.WaitAsync(cancellationToken);
try try
{ {
var current = (await store.ListExpiredLinksAsync(now, cancellationToken)) var current = await store.GetLinkAsync(candidate.Id, cancellationToken);
.SingleOrDefault(link => link.Id == candidate.Id); if (current is null || current.ExpiresAt is null || current.ExpiresAt > now
if (current is null) || !await store.IsEffectiveLinkAsync(current, cancellationToken))
{ {
continue; continue;
} }
LinkPolicy? result;
if (current.DesiredState == "Active") if (current.DesiredState == "Active")
{ {
var result = await DisableCoreAsync( result = await DisableCoreAsync(
current.Id, current.Id,
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()), new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
"system:ttl", "system:ttl",
cancellationToken); cancellationToken);
}
else
{
Publish("link.disconnecting", current);
result = await ConvergeAsync(
current, expectedConnected: false, "system:ttl-retry", cancellationToken);
}
if (result?.ActualState == "Disabled") if (result?.ActualState == "Disabled")
{ {
disabled++; disabled++;
@ -222,29 +246,6 @@ public sealed class LinkService(
{ {
failed++; failed++;
} }
continue;
}
var retrying = await store.SetLinkActualStateAsync(
current.Id, "Disconnecting", null, "system:ttl-retry", cancellationToken) ?? current;
Publish("link.disconnecting", retrying);
try
{
await applier.ApplyDisconnectAsync(retrying, cancellationToken);
await VerifyFactualStateAsync(retrying, expectedConnected: false, cancellationToken);
var completed = await store.SetLinkActualStateAsync(
retrying.Id, "Disabled", null, "system:ttl-retry", cancellationToken) ?? retrying;
Publish("link.disabled", completed);
disabled++;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
var partial = await store.SetLinkActualStateAsync(
retrying.Id, "Partial", CompactError(exception), "system:ttl-retry", cancellationToken)
?? retrying;
Publish("link.partial", partial);
failed++;
}
} }
finally finally
{ {
@ -254,6 +255,119 @@ public sealed class LinkService(
return new LinkExpirationResult(disabled, failed); return new LinkExpirationResult(disabled, failed);
} }
private async Task<LinkPolicy> ConvergeAsync(
LinkPolicy link,
bool expectedConnected,
string actor,
CancellationToken cancellationToken)
{
var current = await store.GetLinkAsync(link.Id, cancellationToken) ?? link;
if ((current.DesiredState == "Active") != expectedConnected
|| !await store.IsEffectiveLinkAsync(current, cancellationToken))
{
return current;
}
var completedState = expectedConnected ? "Active" : "Disabled";
var failureState = expectedConnected ? "Failed" : "Partial";
var completedEvent = expectedConnected ? "link.active" : "link.disabled";
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
try
{
var isConnected = await applier.IsConnectedAsync(current, cancellationToken);
var changedFact = isConnected != expectedConnected;
if (changedFact)
{
var pendingState = expectedConnected ? "Connecting" : "Disconnecting";
if (current.ActualState != pendingState || current.LastError is not null)
{
current = await store.SetLinkActualStateAsync(
current.Id, pendingState, null, actor, cancellationToken) ?? current;
}
if (actor.StartsWith("system:", StringComparison.Ordinal))
{
Publish("link.reconciling", current);
}
if (expectedConnected)
{
await applier.ApplyConnectAsync(current, cancellationToken);
}
else
{
await applier.ApplyDisconnectAsync(current, cancellationToken);
}
await VerifyFactualStateAsync(current, expectedConnected, cancellationToken);
}
if (current.ActualState != completedState || current.LastError is not null)
{
current = await store.SetLinkActualStateAsync(
current.Id, completedState, null, actor, cancellationToken) ?? current;
Publish(completedEvent, current);
}
if (changedFact && actor == "system:reconcile")
{
Publish(expectedConnected ? "link.reapplied" : "link.orphan-removed", current);
}
}
catch (MeshFirewallUnavailableException)
{
current = await store.SetLinkActualStateAsync(
current.Id, failureState, FirewallUnavailableCode, actor, cancellationToken) ?? current;
if (actor != "system:reconcile")
{
Publish(failureEvent, current);
}
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
current = await store.SetLinkActualStateAsync(
current.Id, failureState, CompactError(exception), actor, cancellationToken) ?? current;
Publish(failureEvent, current);
}
return current;
}
private async Task<LinkPolicy?> GetCurrentEffectiveAsync(
string id,
CancellationToken cancellationToken)
{
var current = await store.GetLinkAsync(id, cancellationToken);
if (current is null || !await store.IsEffectiveLinkAsync(current, cancellationToken))
{
return null;
}
return current;
}
private async Task MarkFirewallUnavailableAsync(
IReadOnlyList<LinkPolicy> candidates,
CancellationToken cancellationToken)
{
foreach (var candidate in candidates)
{
using var nodeLease = await AcquireNodeLocksAsync(
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
try
{
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
if (current is not null && IsEligibleForFullReconciliation(current))
{
await store.SetLinkActualStateAsync(
current.Id, "Partial", FirewallUnavailableCode, "system:reconcile", cancellationToken);
}
}
finally
{
gate.Release();
}
}
}
private static bool IsEligibleForFullReconciliation(LinkPolicy link)
=> link.DesiredState == "Active"
|| (link.DesiredState == "Disabled" && link.ActualState != "Disabled");
private void Publish(string type, LinkPolicy link) private void Publish(string type, LinkPolicy link)
=> events.Publish( => events.Publish(
type, type,
@ -289,38 +403,13 @@ public sealed class LinkService(
} }
} }
private async Task<LinkPolicy> ConvergeDisabledCoreAsync( private async Task<SemaphoreSlim> AcquireLinkGateAsync(
LinkPolicy link, string id,
string actor,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var current = (await store.ListLinksAsync(cancellationToken)) var gate = _reconciliationLocks.GetOrAdd(id, static _ => new SemaphoreSlim(1, 1));
.SingleOrDefault(candidate => candidate.Id == link.Id) ?? link; await gate.WaitAsync(cancellationToken);
if (current.DesiredState != "Disabled") return gate;
{
return current;
}
try
{
if (await applier.IsConnectedAsync(current, cancellationToken))
{
await applier.ApplyDisconnectAsync(current, cancellationToken);
await VerifyFactualStateAsync(current, expectedConnected: false, cancellationToken);
}
if (current.ActualState != "Disabled" || current.LastError is not null)
{
current = await store.SetLinkActualStateAsync(
current.Id, "Disabled", null, actor, cancellationToken) ?? current;
Publish("link.disabled", current);
}
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
current = await store.SetLinkActualStateAsync(
current.Id, "Partial", CompactError(exception), actor, cancellationToken) ?? current;
Publish("link.partial", current);
}
return current;
} }
private async Task VerifyFactualStateAsync( private async Task VerifyFactualStateAsync(
@ -336,7 +425,7 @@ public sealed class LinkService(
} }
private static string CompactError(Exception exception) private static string CompactError(Exception exception)
=> exception.Message.Split([(char)13, '\n'], StringSplitOptions.RemoveEmptyEntries) => exception.Message.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
.FirstOrDefault() ?? "Policy application failed."; .FirstOrDefault() ?? "Policy application failed.";
private sealed class LockLease(SemaphoreSlim[] gates) : IDisposable private sealed class LockLease(SemaphoreSlim[] gates) : IDisposable
@ -352,3 +441,4 @@ public sealed class LinkService(
} }
public sealed record LinkExpirationResult(int Disabled, int Failed); public sealed record LinkExpirationResult(int Disabled, int Failed);
public sealed record LinkFullReconciliationResult(int Examined, int Failed, bool FirewallUnavailable);

4
src/ServerMonitorManager.Control/Program.cs
View file

@ -46,9 +46,10 @@ builder.Services.AddOptions<ControlOptions>()
&& options.AuditRetentionDays is >= 1 and <= 3650 && options.AuditRetentionDays is >= 1 and <= 3650
&& options.MaintenanceIntervalMinutes is >= 1 and <= 1440 && options.MaintenanceIntervalMinutes is >= 1 and <= 1440
&& options.LinkExpirationPollSeconds is >= 1 and <= 300 && options.LinkExpirationPollSeconds is >= 1 and <= 300
&& options.LinkReconciliationSeconds is >= 30 and <= 3600
&& options.BackupIntervalHours is >= 1 and <= 720 && options.BackupIntervalHours is >= 1 and <= 720
&& options.BackupRetentionCount is >= 1 and <= 100, && options.BackupRetentionCount is >= 1 and <= 100,
"Invalid Control paths, heartbeat, retention, maintenance, expiration, or backup settings.") "Invalid Control paths, heartbeat, retention, maintenance, expiration, reconciliation, or backup settings.")
.ValidateOnStart(); .ValidateOnStart();
builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSingleton<ControlStore>(); builder.Services.AddSingleton<ControlStore>();
@ -59,6 +60,7 @@ builder.Services.AddSingleton<LinkService>();
builder.Services.AddSingleton<CertificateLifecycleService>(); builder.Services.AddSingleton<CertificateLifecycleService>();
builder.Services.AddSingleton<ControlBackupService>(); builder.Services.AddSingleton<ControlBackupService>();
builder.Services.AddHostedService<LinkExpirationBackgroundService>(); builder.Services.AddHostedService<LinkExpirationBackgroundService>();
builder.Services.AddHostedService<LinkReconciliationBackgroundService>();
builder.Services.AddHostedService<ControlMaintenanceBackgroundService>(); builder.Services.AddHostedService<ControlMaintenanceBackgroundService>();
builder.Services.AddAuthentication(CertificateAuthenticationDefaults.AuthenticationScheme) builder.Services.AddAuthentication(CertificateAuthenticationDefaults.AuthenticationScheme)
.AddCertificate(options => .AddCertificate(options =>

1
src/ServerMonitorManager.Control/appsettings.json
View file

@ -10,6 +10,7 @@
"AuditRetentionDays": 90, "AuditRetentionDays": 90,
"MaintenanceIntervalMinutes": 15, "MaintenanceIntervalMinutes": 15,
"LinkExpirationPollSeconds": 15, "LinkExpirationPollSeconds": 15,
"LinkReconciliationSeconds": 300,
"BackupDirectory": "/var/lib/ochenstarik-server-monitor-manager/backups", "BackupDirectory": "/var/lib/ochenstarik-server-monitor-manager/backups",
"BackupIntervalHours": 24, "BackupIntervalHours": 24,
"BackupRetentionCount": 7, "BackupRetentionCount": 7,

32
src/ServerMonitorManager.Desktop/MainPage.xaml.cs
View file

@ -28,6 +28,7 @@ public sealed partial class MainPage : Page
private SettingsPage? _settingsPage; private SettingsPage? _settingsPage;
private bool _loaded; private bool _loaded;
private bool _controlListening; private bool _controlListening;
private bool _meshFirewallUnavailable;
public MainPage() public MainPage()
{ {
@ -160,7 +161,7 @@ public sealed partial class MainPage : Page
InfoBarSeverity.Warning); InfoBarSeverity.Warning);
} }
if (Servers.Count > 0) if (Servers.Count > 0 || _control.IsConfigured)
{ {
await RefreshAllAsync(); await RefreshAllAsync();
} }
@ -449,6 +450,18 @@ public sealed partial class MainPage : Page
{ {
DispatcherQueue.TryEnqueue(() => DispatcherQueue.TryEnqueue(() =>
{ {
if (controlEvent.Type == MeshLinkViewModel.FirewallUnavailableErrorCode)
{
SetMeshFirewallUnavailable(true);
_ = RefreshMeshAsync(showSuccess: false);
return;
}
if (controlEvent.Type == "mesh.firewall-available")
{
SetMeshFirewallUnavailable(false);
_ = RefreshMeshAsync(showSuccess: false);
return;
}
if (controlEvent.Type.StartsWith("link.", StringComparison.Ordinal)) if (controlEvent.Type.StartsWith("link.", StringComparison.Ordinal))
{ {
ShowInfo( ShowInfo(
@ -730,6 +743,11 @@ public sealed partial class MainPage : Page
{ {
if (Servers.Count == 0) if (Servers.Count == 0)
{ {
if (_control.IsConfigured)
{
await RefreshControlMeshAsync(showSuccess: false);
return;
}
ShowInfo("Серверы не добавлены", "Сначала создайте SSH-ключ и установите его на сервере.", InfoBarSeverity.Informational); ShowInfo("Серверы не добавлены", "Сначала создайте SSH-ключ и установите его на сервере.", InfoBarSeverity.Informational);
return; return;
} }
@ -916,6 +934,12 @@ public sealed partial class MainPage : Page
LinkActionInfo.IsOpen = true; LinkActionInfo.IsOpen = true;
} }
private void SetMeshFirewallUnavailable(bool unavailable)
{
_meshFirewallUnavailable = unavailable;
_linksPage?.SetFirewallUnavailable(unavailable);
}
private ServerViewModel? FindHub() private ServerViewModel? FindHub()
=> Servers.FirstOrDefault(server => server.IsHub); => Servers.FirstOrDefault(server => server.IsHub);
@ -1031,6 +1055,11 @@ public sealed partial class MainPage : Page
link.LastError)); link.LastError));
} }
SetMeshFirewallUnavailable(links
.GroupBy(link => new { link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port })
.Select(group => group.MaxBy(link => link.Version)!)
.Any(link => link.LastError == MeshLinkViewModel.FirewallUnavailableErrorCode));
var activeLinks = MeshLinks.Count(link => link.ActualState == "Active"); var activeLinks = MeshLinks.Count(link => link.ActualState == "Active");
ActiveLinksValueText.Text = activeLinks.ToString(CultureInfo.InvariantCulture); ActiveLinksValueText.Text = activeLinks.ToString(CultureInfo.InvariantCulture);
MeshStatusText.Text = $"Control · {MeshNodes.Count} узлов · {activeLinks} активных · {MeshLinks.Count} политик"; MeshStatusText.Text = $"Control · {MeshNodes.Count} узлов · {activeLinks} активных · {MeshLinks.Count} политик";
@ -1289,6 +1318,7 @@ public sealed partial class MainPage : Page
break; break;
case "links": case "links":
_linksPage ??= new LinksPage(this); _linksPage ??= new LinksPage(this);
_linksPage.SetFirewallUnavailable(_meshFirewallUnavailable);
ShowNavigationPage(_linksPage); ShowNavigationPage(_linksPage);
break; break;
case "sessions": case "sessions":

5
src/ServerMonitorManager.Desktop/MeshModels.cs
View file

@ -19,6 +19,8 @@ public sealed class MeshNodeViewModel
public sealed class MeshLinkViewModel public sealed class MeshLinkViewModel
{ {
public const string FirewallUnavailableErrorCode = "mesh.firewall-unavailable";
public MeshLinkViewModel( public MeshLinkViewModel(
string source, string source,
string target, string target,
@ -63,7 +65,8 @@ public sealed class MeshLinkViewModel
public string DesiredStatusText => $"Желаемое состояние: {DesiredState}"; public string DesiredStatusText => $"Желаемое состояние: {DesiredState}";
public string ActualStatusText => $"Фактическое состояние: {ActualState}"; public string ActualStatusText => $"Фактическое состояние: {ActualState}";
public string DriftText => HasDrift ? "Расхождение: требуется сверка политики" : "Расхождение: нет"; public string DriftText => HasDrift ? "Расхождение: требуется сверка политики" : "Расхождение: нет";
public string ErrorText => string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}"; public string ErrorText => LastError == FirewallUnavailableErrorCode ? string.Empty
: string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}";
public string VersionText => $"Версия политики: {Version}"; public string VersionText => $"Версия политики: {Version}";
public string ExpirationText => ExpiresUnix == 0 public string ExpirationText => ExpiresUnix == 0
? "вручную" ? "вручную"

11
src/ServerMonitorManager.Desktop/Pages/LinksPage.xaml
View file

@ -7,6 +7,7 @@
<Grid RowSpacing="16"> <Grid RowSpacing="16">
<Grid.RowDefinitions> <Grid.RowDefinitions>
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" /> <RowDefinition Height="Auto" />
<RowDefinition Height="*" /> <RowDefinition Height="*" />
</Grid.RowDefinitions> </Grid.RowDefinitions>
@ -27,7 +28,15 @@
</CommandBar> </CommandBar>
</Grid> </Grid>
<Grid x:Name="LinksWorkspace" Grid.Row="1" ColumnSpacing="20" RowSpacing="20"> <InfoBar
x:Name="FirewallUnavailableInfo"
Grid.Row="1"
IsClosable="False"
IsOpen="False"
Message="Mesh firewall не загружен"
Severity="Error" />
<Grid x:Name="LinksWorkspace" Grid.Row="2" ColumnSpacing="20" RowSpacing="20">
<Grid.ColumnDefinitions> <Grid.ColumnDefinitions>
<ColumnDefinition x:Name="EditorColumn" Width="*" /> <ColumnDefinition x:Name="EditorColumn" Width="*" />
<ColumnDefinition x:Name="ListColumn" Width="0" /> <ColumnDefinition x:Name="ListColumn" Width="0" />

3
src/ServerMonitorManager.Desktop/Pages/LinksPage.xaml.cs
View file

@ -18,6 +18,9 @@ public sealed partial class LinksPage : Page
public ObservableCollection<MeshLinkViewModel> Links => _host.MeshLinks; public ObservableCollection<MeshLinkViewModel> Links => _host.MeshLinks;
internal void SetFirewallUnavailable(bool unavailable)
=> FirewallUnavailableInfo.IsOpen = unavailable;
private async void RefreshButton_Click(object sender, RoutedEventArgs e) private async void RefreshButton_Click(object sender, RoutedEventArgs e)
=> await _host.RefreshLinksFromPageAsync(); => await _host.RefreshLinksFromPageAsync();

35
tests/ServerMonitorManager.Control.Tests/LinkPolicyApplierIntegrationTests.cs
View file

@ -26,6 +26,7 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
var sudoPath = Path.Combine(_directory, "sudo"); var sudoPath = Path.Combine(_directory, "sudo");
var helperPath = Path.Combine(_directory, "policy-helper"); var helperPath = Path.Combine(_directory, "policy-helper");
var failureMarkerPath = Path.Combine(_directory, "fail-disconnect"); var failureMarkerPath = Path.Combine(_directory, "fail-disconnect");
var firewallUnavailableMarkerPath = Path.Combine(_directory, "firewall-unavailable");
var connectedMarkerPath = Path.Combine(_directory, "connected"); var connectedMarkerPath = Path.Combine(_directory, "connected");
var invocationLogPath = Path.Combine(_directory, "helper.log"); var invocationLogPath = Path.Combine(_directory, "helper.log");
await WriteExecutableAsync( await WriteExecutableAsync(
@ -58,6 +59,10 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
rm -f '{{ShellQuote(connectedMarkerPath)}}' rm -f '{{ShellQuote(connectedMarkerPath)}}'
fi fi
if [ "${1:-}" = "link-status" ]; then if [ "${1:-}" = "link-status" ]; then
if [ -f '{{ShellQuote(firewallUnavailableMarkerPath)}}' ]; then
echo "mesh.firewall-unavailable" >&2
exit 79
fi
if [ -f '{{ShellQuote(connectedMarkerPath)}}' ]; then if [ -f '{{ShellQuote(connectedMarkerPath)}}' ]; then
printf '%s\n' active printf '%s\n' active
else else
@ -79,6 +84,28 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
cancellationToken); cancellationToken);
Assert.Equal("Active", active.ActualState); Assert.Equal("Active", active.ActualState);
File.Delete(connectedMarkerPath);
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
var restored = await service.ReconcileAllAsync(cancellationToken);
Assert.False(restored.FirewallUnavailable);
Assert.Equal("Active", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
Assert.Equal(
[
"link-status ai-agent home tcp 22",
"link-connect ai-agent home tcp 22 60",
"link-status ai-agent home tcp 22"
],
await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
await File.WriteAllTextAsync(firewallUnavailableMarkerPath, "fail", cancellationToken);
var unavailable = await service.ReconcileAllAsync(cancellationToken);
Assert.True(unavailable.FirewallUnavailable);
Assert.Equal(LinkService.FirewallUnavailableCode,
(await store.GetLinkAsync(active.Id, cancellationToken))!.LastError);
File.Delete(firewallUnavailableMarkerPath);
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken); await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken);
var partial = await service.DisableAsync( var partial = await service.DisableAsync(
active.Id, active.Id,
@ -110,16 +137,14 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
Assert.Equal("Disabled", persisted.ActualState); Assert.Equal("Disabled", persisted.ActualState);
var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken); var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken);
Assert.Equal(9, invocations.Length); Assert.Equal(7, invocations.Length);
Assert.Equal("link-connect ai-agent home tcp 22 60", invocations[0]); Assert.Equal("link-status ai-agent home tcp 22", invocations[0]);
Assert.Equal("link-status ai-agent home tcp 22", invocations[1]); Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[1]);
Assert.Equal("link-status ai-agent home tcp 22", invocations[2]); Assert.Equal("link-status ai-agent home tcp 22", invocations[2]);
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[3]); Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[3]);
Assert.Equal("link-status ai-agent home tcp 22", invocations[4]); Assert.Equal("link-status ai-agent home tcp 22", invocations[4]);
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[5]); Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[5]);
Assert.Equal("link-status ai-agent home tcp 22", invocations[6]); Assert.Equal("link-status ai-agent home tcp 22", invocations[6]);
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[7]);
Assert.Equal("link-status ai-agent home tcp 22", invocations[8]);
} }
public ValueTask DisposeAsync() public ValueTask DisposeAsync()

338
tests/ServerMonitorManager.Control.Tests/LinkReconciliationTests.cs Normal file
View file

@ -0,0 +1,338 @@
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ServerMonitorManager.Control;
using ServerMonitorManager.Core;
using Xunit;
namespace ServerMonitorManager.Control.Tests;
public sealed class LinkReconciliationTests : IAsyncDisposable
{
private readonly string _directory = Path.Combine(
Path.GetTempPath(), $"smm-link-reconciliation-{Guid.NewGuid():N}");
[Fact]
public async Task AllPolicyPassReappliesErasedActiveRulesWithoutHeartbeat()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "AA11", cancellationToken);
await EnrollAgentAsync(store, "target-one", "BB22", cancellationToken);
await EnrollAgentAsync(store, "target-two", "CC33", cancellationToken);
var applier = new RecordingPolicyApplier();
var service = new LinkService(store, applier, new ControlEventBroker());
var first = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
var second = await service.CreateAsync(CreateRequest("target-two"), "operator", cancellationToken);
applier.EraseRules();
var mutationsBefore = applier.ConnectCalls;
var result = await service.ReconcileAllAsync(cancellationToken);
Assert.Equal(2, result.Examined);
Assert.Equal(0, result.Failed);
Assert.Equal(mutationsBefore + 2, applier.ConnectCalls);
Assert.All(await store.ListEffectiveLinksAsync(cancellationToken),
link => Assert.Equal("Active", link.ActualState));
Assert.True(applier.IsConnected(first));
Assert.True(applier.IsConnected(second));
}
[Fact]
public async Task SecondUnchangedPassDoesNotMutateFirewall()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "A111", cancellationToken);
await EnrollAgentAsync(store, "target-one", "B222", cancellationToken);
var applier = new RecordingPolicyApplier();
var service = new LinkService(store, applier, new ControlEventBroker());
await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
await service.ReconcileAllAsync(cancellationToken);
var connects = applier.ConnectCalls;
var disconnects = applier.DisconnectCalls;
await service.ReconcileAllAsync(cancellationToken);
Assert.Equal(connects, applier.ConnectCalls);
Assert.Equal(disconnects, applier.DisconnectCalls);
}
[Fact]
public async Task AllPolicyPassRemovesOrphanRuleForDesiredDisabledLink()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "DD44", cancellationToken);
await EnrollAgentAsync(store, "target-one", "EE55", cancellationToken);
var broker = new ControlEventBroker();
using var subscription = broker.Subscribe();
var applier = new RecordingPolicyApplier();
var service = new LinkService(store, applier, broker);
var active = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
await store.BeginDisableLinkMutationAsync(
active.Id,
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
"operator",
cancellationToken);
await service.ReconcileAllAsync(cancellationToken);
Assert.Equal(1, applier.DisconnectCalls);
Assert.Equal("Disabled", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
var eventTypes = new List<string>();
while (subscription.Reader.TryRead(out var controlEvent))
{
eventTypes.Add(controlEvent.Type);
}
Assert.Contains("link.orphan-removed", eventTypes);
}
[Fact]
public async Task FirewallUnavailableAbortsWithoutMutationsAndPublishesOneSharedEvent()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "FF66", cancellationToken);
await EnrollAgentAsync(store, "target-one", "0011", cancellationToken);
await EnrollAgentAsync(store, "target-two", "0022", cancellationToken);
var broker = new ControlEventBroker();
using var subscription = broker.Subscribe();
var applier = new RecordingPolicyApplier();
var service = new LinkService(store, applier, broker);
await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
await service.CreateAsync(CreateRequest("target-two"), "operator", cancellationToken);
while (subscription.Reader.TryRead(out _)) { }
var connectBefore = applier.ConnectCalls;
var disconnectBefore = applier.DisconnectCalls;
applier.FirewallUnavailable = true;
var result = await service.ReconcileAllAsync(cancellationToken);
Assert.True(result.FirewallUnavailable);
Assert.Equal(connectBefore, applier.ConnectCalls);
Assert.Equal(disconnectBefore, applier.DisconnectCalls);
Assert.All(await store.ListEffectiveLinksAsync(cancellationToken), link =>
{
Assert.Equal("Partial", link.ActualState);
Assert.Equal(LinkService.FirewallUnavailableCode, link.LastError);
});
var eventTypes = new List<string>();
while (subscription.Reader.TryRead(out var controlEvent))
{
eventTypes.Add(controlEvent.Type);
}
Assert.Equal([LinkService.FirewallUnavailableCode], eventTypes);
applier.FirewallUnavailable = false;
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
eventTypes.Clear();
while (subscription.Reader.TryRead(out var controlEvent))
{
eventTypes.Add(controlEvent.Type);
}
Assert.Single(eventTypes, eventType => eventType == LinkService.FirewallAvailableCode);
}
[Fact]
public async Task MarkerCreatedAfterNormalPassTriggersPromptAdditionalPass()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "A333", cancellationToken);
await EnrollAgentAsync(store, "target-one", "B444", cancellationToken);
var applier = new RecordingPolicyApplier();
var links = new LinkService(store, applier, new ControlEventBroker());
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
var background = CreateBackgroundService(
links,
applier,
new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z")));
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
Assert.Equal(0, applier.CompleteReconciliationCalls);
applier.ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
Assert.Equal(2, applier.ReconciliationStatusCalls);
Assert.Equal(1, applier.CompleteReconciliationCalls);
Assert.Null(applier.ReconciliationRequest);
Assert.Null(await background.RunOnceAsync(cancellationToken));
}
[Fact]
public async Task CompletingObservedGenerationDoesNotConsumeNewerRequest()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "A355", cancellationToken);
await EnrollAgentAsync(store, "target-one", "B466", cancellationToken);
var generationA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
var generationB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
var applier = new RecordingPolicyApplier { ReconciliationRequest = generationA };
var links = new LinkService(store, applier, new ControlEventBroker());
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
applier.ReplacementRequestOnNextProbe = generationB;
var background = CreateBackgroundService(
links,
applier,
new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z")));
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
Assert.Equal([generationA], applier.CompletedGenerations);
Assert.Equal(generationB, applier.ReconciliationRequest);
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
Assert.Equal([generationA, generationB], applier.CompletedGenerations);
Assert.Null(applier.ReconciliationRequest);
}
[Fact]
public async Task BackgroundPassRetainsMarkerWhenFirewallIsUnavailable()
{
var cancellationToken = TestContext.Current.CancellationToken;
var store = CreateStore();
await store.InitializeAsync(cancellationToken);
await EnrollAgentAsync(store, "source", "A555", cancellationToken);
await EnrollAgentAsync(store, "target-one", "B666", cancellationToken);
var applier = new RecordingPolicyApplier
{
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
};
var links = new LinkService(store, applier, new ControlEventBroker());
await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
applier.FirewallUnavailable = true;
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z"));
var background = CreateBackgroundService(links, applier, time);
var result = await background.RunOnceAsync(cancellationToken);
Assert.NotNull(result);
Assert.True(result.FirewallUnavailable);
Assert.Equal(0, applier.CompleteReconciliationCalls);
Assert.NotNull(applier.ReconciliationRequest);
applier.FirewallUnavailable = false;
Assert.Null(await background.RunOnceAsync(cancellationToken));
time.Advance(TimeSpan.FromSeconds(30));
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
Assert.Null(applier.ReconciliationRequest);
}
public ValueTask DisposeAsync()
{
SqliteConnection.ClearAllPools();
if (Directory.Exists(_directory))
{
Directory.Delete(_directory, recursive: true);
}
return ValueTask.CompletedTask;
}
private ControlStore CreateStore()
{
Directory.CreateDirectory(_directory);
return new ControlStore(Options.Create(new ControlOptions
{
DatabasePath = Path.Combine(_directory, "control.db"),
CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx")
}));
}
private static LinkPolicyCreateRequest CreateRequest(string target)
=> new("source", target, "tcp", 22, 0, "test", Guid.NewGuid().ToString());
private static LinkReconciliationBackgroundService CreateBackgroundService(
LinkService links,
ILinkPolicyApplier applier,
TimeProvider timeProvider)
=> new(
links,
applier,
Options.Create(new ControlOptions { LinkReconciliationSeconds = 30 }),
timeProvider,
NullLogger<LinkReconciliationBackgroundService>.Instance);
private static async Task EnrollAgentAsync(
ControlStore store,
string nodeId,
string thumbprint,
CancellationToken cancellationToken)
{
var token = await store.CreateEnrollmentTokenAsync(nodeId, TimeSpan.FromMinutes(10), cancellationToken);
Assert.NotNull(await store.EnrollAsync(
new EnrollmentRequest(nodeId, token, "csr", Guid.NewGuid().ToString()),
() => new IssuedCertificate("certificate", "ca", thumbprint, DateTimeOffset.UtcNow.AddYears(1)),
cancellationToken));
}
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
{
private readonly HashSet<string> _connected = [];
public int ConnectCalls { get; private set; }
public int DisconnectCalls { get; private set; }
public bool FirewallUnavailable { get; set; }
public string? ReconciliationRequest { get; set; }
public string? ReplacementRequestOnNextProbe { get; set; }
public int ReconciliationStatusCalls { get; private set; }
public int CompleteReconciliationCalls { get; private set; }
public List<string> CompletedGenerations { get; } = [];
public void EraseRules() => _connected.Clear();
public bool IsConnected(LinkPolicy link) => _connected.Contains(link.Id);
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
{
ConnectCalls++;
_connected.Add(link.Id);
return Task.CompletedTask;
}
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
{
DisconnectCalls++;
_connected.Remove(link.Id);
return Task.CompletedTask;
}
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
{
if (ReplacementRequestOnNextProbe is not null)
{
ReconciliationRequest = ReplacementRequestOnNextProbe;
ReplacementRequestOnNextProbe = null;
}
return FirewallUnavailable
? Task.FromException<bool>(new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode))
: Task.FromResult(IsConnected(link));
}
public Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
{
ReconciliationStatusCalls++;
return Task.FromResult(ReconciliationRequest);
}
public Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
{
CompleteReconciliationCalls++;
CompletedGenerations.Add(generation);
if (ReconciliationRequest == generation)
{
ReconciliationRequest = null;
}
return Task.CompletedTask;
}
}
private sealed class TestTimeProvider(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;
public void Advance(TimeSpan value) => now += value;
}
}

81
tests/acceptance/three-server-mesh.sh
View file

@ -22,6 +22,7 @@ SOURCE_NODE_ID="${SOURCE_NODE_ID:-ai-agent}"
HOME_NODE_ID="${HOME_NODE_ID:-home}" HOME_NODE_ID="${HOME_NODE_ID:-home}"
SECOND_NODE_ID="${SECOND_NODE_ID:-second}" SECOND_NODE_ID="${SECOND_NODE_ID:-second}"
TARGET_PORT="${TARGET_PORT:-22}" TARGET_PORT="${TARGET_PORT:-22}"
LINK_RECONCILIATION_SECONDS="${LINK_RECONCILIATION_SECONDS:-300}"
CONTROL_DEVICE_ID="acceptance-$(date +%s)" CONTROL_DEVICE_ID="acceptance-$(date +%s)"
INSTALLER_COMMAND="${INSTALLER_COMMAND:-sudo /usr/local/sbin/ochenstarik-server-monitor-manager.sh}" INSTALLER_COMMAND="${INSTALLER_COMMAND:-sudo /usr/local/sbin/ochenstarik-server-monitor-manager.sh}"
WORK_DIRECTORY="$(mktemp -d)" WORK_DIRECTORY="$(mktemp -d)"
@ -48,31 +49,54 @@ source_ssh() {
"$SOURCE_SSH_USER@$SOURCE_SSH_HOST" "$@" "$SOURCE_SSH_USER@$SOURCE_SSH_HOST" "$@"
} }
expect_reachable() { probe_reachable() {
local ip="$1" local ip="$1"
source_ssh "nc -z -w 5 '$ip' '$TARGET_PORT'" source_ssh "nc -z -w 5 '$ip' '$TARGET_PORT'"
} }
expect_reachable() {
local ip="$1"
probe_reachable "$ip" || {
echo "Expected access from $SOURCE_NODE_ID to $ip:$TARGET_PORT" >&2
exit 1
}
}
probe_blocked() {
local ip="$1"
! probe_reachable "$ip"
}
expect_blocked() { expect_blocked() {
local ip="$1" local ip="$1"
if source_ssh "nc -z -w 5 '$ip' '$TARGET_PORT'"; then if ! probe_blocked "$ip"; then
echo "Unexpected access from $SOURCE_NODE_ID to $ip:$TARGET_PORT" >&2 echo "Unexpected access from $SOURCE_NODE_ID to $ip:$TARGET_PORT" >&2
exit 1 exit 1
fi fi
} }
expect_factual_status() { probe_factual_status() {
local target="$1" local target="$1"
local expected="$2" local expected="$2"
local command output local command output
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \ printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT" link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
output="$(hub_ssh "$command")" output="$(hub_ssh "$command")"
[[ "$output" == "$expected" ]] || { [[ "$output" == "$expected" ]]
}
expect_factual_status() {
local target="$1"
local expected="$2"
if ! probe_factual_status "$target" "$expected"; then
local command output
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
output="$(hub_ssh "$command")"
echo "Unexpected factual Link status for $SOURCE_NODE_ID -> $target: $output (expected $expected)" >&2 echo "Unexpected factual Link status for $SOURCE_NODE_ID -> $target: $output (expected $expected)" >&2
exit 1 exit 1
fi
} }
}
decode_base64url() { decode_base64url() {
local value="${1//-/+}" local value="${1//-/+}"
@ -85,11 +109,11 @@ decode_base64url() {
printf '%s' "$value" | base64 --decode printf '%s' "$value" | base64 --decode
} }
echo '[1/11] Checking installed services on all three nodes' echo '[1/12] Checking installed services on all three nodes'
hub_ssh "sudo systemctl is-active ochenstarik-smm-control.service >/dev/null" hub_ssh "sudo systemctl is-active ochenstarik-smm-control.service >/dev/null"
source_ssh "sudo systemctl is-active ochenstarik-smm-agent.service >/dev/null" source_ssh "sudo systemctl is-active ochenstarik-smm-agent.service >/dev/null"
echo '[2/11] Creating an isolated operator identity for this acceptance run' echo '[2/12] Creating an isolated operator identity for this acceptance run'
device_code="$(hub_ssh "$INSTALLER_COMMAND control-device-code '$CONTROL_DEVICE_ID'" | tr -d '\r' | grep -o 'SMMDEV1-[A-Za-z0-9_-]*' | tail -n1)" device_code="$(hub_ssh "$INSTALLER_COMMAND control-device-code '$CONTROL_DEVICE_ID'" | tr -d '\r' | grep -o 'SMMDEV1-[A-Za-z0-9_-]*' | tail -n1)"
[[ -n "$device_code" ]] || { echo 'Hub did not return SMMDEV1 code' >&2; exit 1; } [[ -n "$device_code" ]] || { echo 'Hub did not return SMMDEV1 code' >&2; exit 1; }
decode_base64url "${device_code#SMMDEV1-}" >"$WORK_DIRECTORY/device.env" decode_base64url "${device_code#SMMDEV1-}" >"$WORK_DIRECTORY/device.env"
@ -147,7 +171,7 @@ disable_link() {
"$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" '{idempotencyKey:$key}')" "$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" '{idempotencyKey:$key}')"
} }
echo '[3/11] Confirming all expected Agent identities are online' echo '[3/12] Confirming all expected Agent identities are online'
agents="$(api_get '/api/v1/control/agents')" agents="$(api_get '/api/v1/control/agents')"
for node in "$SOURCE_NODE_ID" "$HOME_NODE_ID" "$SECOND_NODE_ID"; do for node in "$SOURCE_NODE_ID" "$HOME_NODE_ID" "$SECOND_NODE_ID"; do
jq -e --arg node "$node" '.[] | select(.nodeId == $node)' <<<"$agents" >/dev/null || { jq -e --arg node "$node" '.[] | select(.nodeId == $node)' <<<"$agents" >/dev/null || {
@ -156,7 +180,7 @@ for node in "$SOURCE_NODE_ID" "$HOME_NODE_ID" "$SECOND_NODE_ID"; do
} }
done done
echo '[4/11] Creating independent Links to home and second server' echo '[4/12] Creating independent Links to home and second server'
home_link="$(create_link "$HOME_NODE_ID" 0)" home_link="$(create_link "$HOME_NODE_ID" 0)"
second_link="$(create_link "$SECOND_NODE_ID" 0)" second_link="$(create_link "$SECOND_NODE_ID" 0)"
LINK_HOME_ID="$(jq -r '.id' <<<"$home_link")" LINK_HOME_ID="$(jq -r '.id' <<<"$home_link")"
@ -166,18 +190,18 @@ jq -e '.desiredState == "Active" and .actualState == "Active" and .lastError ==
expect_factual_status "$HOME_NODE_ID" active expect_factual_status "$HOME_NODE_ID" active
expect_factual_status "$SECOND_NODE_ID" active expect_factual_status "$SECOND_NODE_ID" active
echo '[5/11] Verifying routed access through both Links' echo '[5/12] Verifying routed access through both Links'
expect_reachable "$HOME_WG_IP" expect_reachable "$HOME_WG_IP"
expect_reachable "$SECOND_WG_IP" expect_reachable "$SECOND_WG_IP"
echo '[6/11] Disabling only the second Link' echo '[6/12] Disabling only the second Link'
disable_link "$LINK_SECOND_ID" | jq -e \ disable_link "$LINK_SECOND_ID" | jq -e \
'.desiredState == "Disabled" and .actualState == "Disabled" and .lastError == null' >/dev/null '.desiredState == "Disabled" and .actualState == "Disabled" and .lastError == null' >/dev/null
expect_factual_status "$SECOND_NODE_ID" disabled expect_factual_status "$SECOND_NODE_ID" disabled
expect_reachable "$HOME_WG_IP" expect_reachable "$HOME_WG_IP"
expect_blocked "$SECOND_WG_IP" expect_blocked "$SECOND_WG_IP"
echo '[7/11] Verifying automatic TTL expiration' echo '[7/12] Verifying automatic TTL expiration'
ttl_link="$(create_link "$SECOND_NODE_ID" 1)" ttl_link="$(create_link "$SECOND_NODE_ID" 1)"
ttl_id="$(jq -r '.id' <<<"$ttl_link")" ttl_id="$(jq -r '.id' <<<"$ttl_link")"
expect_reachable "$SECOND_WG_IP" expect_reachable "$SECOND_WG_IP"
@ -196,13 +220,34 @@ expect_factual_status "$SECOND_NODE_ID" disabled
expect_blocked "$SECOND_WG_IP" expect_blocked "$SECOND_WG_IP"
expect_reachable "$HOME_WG_IP" expect_reachable "$HOME_WG_IP"
echo '[8/11] Creating and validating a Control backup' if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
echo '[8/12] Restoring the base firewall without restarting any Node'
hub_ssh 'sudo /usr/local/sbin/ochenstarik-smm-emergency firewall-restore'
deadline=$((SECONDS + LINK_RECONCILIATION_SECONDS + 30))
while ((SECONDS < deadline)); do
if probe_factual_status "$HOME_NODE_ID" active \
&& probe_factual_status "$SECOND_NODE_ID" disabled \
&& probe_reachable "$HOME_WG_IP" \
&& probe_blocked "$SECOND_WG_IP"; then
break
fi
sleep 5
done
expect_factual_status "$HOME_NODE_ID" active
expect_factual_status "$SECOND_NODE_ID" disabled
expect_reachable "$HOME_WG_IP"
expect_blocked "$SECOND_WG_IP"
else
echo '[8/12] Firewall restore reconciliation skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
fi
echo '[9/12] Creating and validating a Control backup'
backup_path="$(hub_ssh "sudo systemd-run --wait --pipe --quiet --collect --uid=ochenstarik-smm-control --gid=ochenstarik-smm-control -p EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-create" | grep '/backup-' | tail -n1)" backup_path="$(hub_ssh "sudo systemd-run --wait --pipe --quiet --collect --uid=ochenstarik-smm-control --gid=ochenstarik-smm-control -p EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-create" | grep '/backup-' | tail -n1)"
[[ "$backup_path" == */backup-* ]] || { echo 'Backup command did not return a backup path' >&2; exit 1; } [[ "$backup_path" == */backup-* ]] || { echo 'Backup command did not return a backup path' >&2; exit 1; }
hub_ssh "sudo test -s '$backup_path/manifest.json' && sudo test -s '$backup_path/control.db' && sudo test -s '$backup_path/control-ca.pfx'" hub_ssh "sudo test -s '$backup_path/manifest.json' && sudo test -s '$backup_path/control.db' && sudo test -s '$backup_path/control-ca.pfx'"
if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
echo '[9/11] Restoring the verified backup and restarting Control' echo '[10/12] Restoring the verified backup and restarting Control'
hub_ssh "sudo sh -c 'set -a; . /etc/ochenstarik-server-monitor-manager/control.env; set +a; systemctl stop ochenstarik-smm-control.service; /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-restore \"\$1\"; status=\$?; systemctl start ochenstarik-smm-control.service; exit \$status' sh '$backup_path'" hub_ssh "sudo sh -c 'set -a; . /etc/ochenstarik-server-monitor-manager/control.env; set +a; systemctl stop ochenstarik-smm-control.service; /usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control backup-restore \"\$1\"; status=\$?; systemctl start ochenstarik-smm-control.service; exit \$status' sh '$backup_path'"
for _ in {1..30}; do for _ in {1..30}; do
if api_get '/healthz' >/dev/null 2>&1; then if api_get '/healthz' >/dev/null 2>&1; then
@ -216,11 +261,11 @@ if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
expect_factual_status "$HOME_NODE_ID" active expect_factual_status "$HOME_NODE_ID" active
expect_factual_status "$SECOND_NODE_ID" disabled expect_factual_status "$SECOND_NODE_ID" disabled
else else
echo '[9/11] Restore check skipped; set SMM_ACCEPT_RESTORE=1 to enable it' echo '[10/12] Restore check skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
fi fi
if [[ "${SMM_ACCEPT_REBOOT:-0}" == '1' ]]; then if [[ "${SMM_ACCEPT_REBOOT:-0}" == '1' ]]; then
echo '[10/11] Rebooting the Hub and source Node and rechecking policy state' echo '[11/12] Rebooting the Hub and source Node and rechecking policy state'
hub_ssh 'sudo systemctl reboot' || true hub_ssh 'sudo systemctl reboot' || true
source_ssh 'sudo systemctl reboot' || true source_ssh 'sudo systemctl reboot' || true
sleep 15 sleep 15
@ -235,10 +280,10 @@ if [[ "${SMM_ACCEPT_REBOOT:-0}" == '1' ]]; then
expect_factual_status "$HOME_NODE_ID" active expect_factual_status "$HOME_NODE_ID" active
expect_factual_status "$SECOND_NODE_ID" disabled expect_factual_status "$SECOND_NODE_ID" disabled
else else
echo '[10/11] Reboot check skipped; set SMM_ACCEPT_REBOOT=1 to enable it' echo '[11/12] Reboot check skipped; set SMM_ACCEPT_REBOOT=1 to enable it'
fi fi
echo '[11/11] Revoking the temporary Operator certificate' echo '[12/12] Revoking the temporary Operator certificate'
api_post "/api/v1/control/devices/$CONTROL_DEVICE_ID/reenroll" \ api_post "/api/v1/control/devices/$CONTROL_DEVICE_ID/reenroll" \
"$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" \ "$(jq -cn --arg key "$(cat /proc/sys/kernel/random/uuid)" \
'{reason:"three-server acceptance completed",idempotencyKey:$key}')" \ '{reason:"three-server acceptance completed",idempotencyKey:$key}')" \

86
tests/bootstrap/test-bootstrap-contract.sh
View file

@ -6,11 +6,13 @@ root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh" bootstrap="$root/deploy/ochenstarik-server-monitor-manager.sh"
helper="$root/deploy/ochenstarik-smm-policy-apply" helper="$root/deploy/ochenstarik-smm-policy-apply"
emergency="$root/deploy/ochenstarik-smm-emergency" emergency="$root/deploy/ochenstarik-smm-emergency"
acceptance="$root/tests/acceptance/three-server-mesh.sh"
grep -Fq 'if ! listing="$(/usr/sbin/nft -a list chain' "$helper" || { grep -Fq 'listing="$(/usr/sbin/nft -a list chain' "$helper" || {
printf '%s\n' "policy status probe must fail closed when nftables cannot be inspected" >&2 printf '%s\n' "policy status probe must fail closed when nftables cannot be inspected" >&2
exit 1 exit 1
} }
grep -Fq "grep -Eiq 'No such file or directory|does not exist'" "$helper"
provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service" provisioning_helper_unit="$root/deploy/ochenstarik-smm-provisioning-helper.service"
grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit" grep -Fq 'EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env' "$provisioning_helper_unit"
@ -61,6 +63,13 @@ grep -Eq '^ochenstarik-server-monitor-manager [0-9]+\.[0-9]+\.[0-9]+-' <<<"$vers
emergency_help="$(bash "$emergency" --help)" emergency_help="$(bash "$emergency" --help)"
grep -Fq 'mesh-disable' <<<"$emergency_help" grep -Fq 'mesh-disable' <<<"$emergency_help"
grep -Fq 'firewall-restore' <<<"$emergency_help" grep -Fq 'firewall-restore' <<<"$emergency_help"
grep -Fq 'readonly RECONCILE_MARKER="$STATE_DIR/mesh/reconcile-requested"' "$emergency"
grep -Fq 'chown root:root "$temporary_marker"' "$emergency"
grep -Fq 'chmod 0600 "$temporary_marker"' "$emergency"
grep -Fq 'mv -f -- "$temporary_marker" "$RECONCILE_MARKER"' "$emergency"
grep -Fq '/usr/bin/flock -x 9' "$emergency"
grep -Fq 'generation="$(</proc/sys/kernel/random/uuid)"' "$emergency"
[[ "$(grep -Fc ' request_reconciliation' "$emergency")" -ge 2 ]]
if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then if bash "$bootstrap" unsupported-action >/dev/null 2>&1; then
printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2 printf '%s\n' "unsupported bootstrap action unexpectedly succeeded" >&2
@ -97,8 +106,83 @@ if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
printf '%s\n' "policy helper unexpectedly accepted extra link-status arguments" >&2 printf '%s\n' "policy helper unexpectedly accepted extra link-status arguments" >&2
exit 1 exit 1
fi fi
generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)"
printf '%s\n' "$generation_a" >"$reconcile_marker"
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
bash "$helper" reconcile-status)" == "requested:$generation_a" ]]
printf '%s\n' "$generation_b" >"$reconcile_marker"
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
bash "$helper" reconcile-complete "$generation_a" >/dev/null
[[ "$(<"$reconcile_marker")" == "$generation_b" ]]
SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
bash "$helper" reconcile-complete "$generation_b" >/dev/null
[[ ! -e "$reconcile_marker" ]]
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
bash "$helper" reconcile-status)" == 'complete' ]]
if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then
printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2
exit 1
fi
firewall_error="$(mktemp -t smm-firewall-error.XXXXXXXX)"
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
SMM_POLICY_FIREWALL_UNAVAILABLE=1 bash "$helper" \
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
printf '%s\n' "missing firewall unexpectedly produced a factual Link status" >&2
exit 1
else
[[ $? -eq 79 ]]
fi
[[ "$(<"$firewall_error")" == 'mesh.firewall-unavailable' ]]
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
SMM_POLICY_FIREWALL_ERROR='permission denied' bash "$helper" \
link-status source target tcp 22 >/dev/null 2>"$firewall_error"; then
printf '%s\n' "unknown nft inspection error unexpectedly produced a factual Link status" >&2
exit 1
else
[[ $? -eq 78 ]]
fi
grep -Fq 'permission denied' "$firewall_error"
if grep -Fq 'mesh.firewall-unavailable' "$firewall_error"; then
printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2
exit 1
fi
rm -f -- "$firewall_error" "$reconcile_marker"
rm -f -- "$policy_state" rm -f -- "$policy_state"
extract_shell_function() {
local name="$1"
awk -v signature="$name() {" '
$0 == signature { emitting = 1 }
emitting { print }
emitting && $0 == "}" { exit }
' "$acceptance"
}
eval "$(extract_shell_function probe_factual_status)"
SOURCE_NODE_ID=source
TARGET_PORT=22
probe_counter="$(mktemp -t smm-factual-probe.XXXXXXXX)"
printf '%s\n' 0 >"$probe_counter"
hub_ssh() {
local count
count="$(( $(<"$probe_counter") + 1 ))"
printf '%s\n' "$count" >"$probe_counter"
[[ "$count" -eq 1 ]] && printf '%s\n' disabled || printf '%s\n' active
}
if probe_factual_status target active; then
printf '%s\n' "initial factual mismatch unexpectedly passed" >&2
exit 1
fi
probe_factual_status target active || {
printf '%s\n' "factual probe did not allow convergence after an initial mismatch" >&2
exit 1
}
rm -f -- "$probe_counter" "${reconcile_marker}.lock"
fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)" fixture="$(mktemp -d -t smm-bootstrap-test.XXXXXXXX)"
trap 'rm -rf -- "$fixture"' EXIT trap 'rm -rf -- "$fixture"' EXIT
mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap" mkdir -p "$fixture/payload/agent" "$fixture/payload/control" "$fixture/payload/provisioning-helper" "$fixture/payload/deploy" "$fixture/payload/bootstrap"

18
tests/windows/Test-DesktopContracts.ps1
View file

@ -19,6 +19,8 @@ $linksCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
Join-Path $root 'src\ServerMonitorManager.Desktop\Pages\LinksPage.xaml.cs') Join-Path $root 'src\ServerMonitorManager.Desktop\Pages\LinksPage.xaml.cs')
$mainCode = Get-Content -Raw -Encoding UTF8 -LiteralPath ( $mainCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
Join-Path $root 'src\ServerMonitorManager.Desktop\MainPage.xaml.cs') Join-Path $root 'src\ServerMonitorManager.Desktop\MainPage.xaml.cs')
$meshModelsCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
Join-Path $root 'src\ServerMonitorManager.Desktop\MeshModels.cs')
$appCode = Get-Content -Raw -Encoding UTF8 -LiteralPath ( $appCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
Join-Path $root 'src\ServerMonitorManager.Desktop\App.xaml.cs') Join-Path $root 'src\ServerMonitorManager.Desktop\App.xaml.cs')
$sshCode = Get-Content -Raw -Encoding UTF8 -LiteralPath ( $sshCode = Get-Content -Raw -Encoding UTF8 -LiteralPath (
@ -52,6 +54,22 @@ if ($linksCode.IndexOf(
'LinksList.SelectedItem as MeshLinkViewModel', [StringComparison]::Ordinal) -lt 0) { 'LinksList.SelectedItem as MeshLinkViewModel', [StringComparison]::Ordinal) -lt 0) {
throw 'Links page must pass its selected Link to the command handler.' throw 'Links page must pass its selected Link to the command handler.'
} }
if ($linksXaml.IndexOf('x:Name="FirewallUnavailableInfo"', [StringComparison]::Ordinal) -lt 0 -or
$linksXaml.IndexOf('Message="Mesh firewall ', [StringComparison]::Ordinal) -lt 0) {
throw 'Links page must expose one persistent Mesh firewall unavailable banner.'
}
if ($linksCode.IndexOf('SetFirewallUnavailable', [StringComparison]::Ordinal) -lt 0 -or
$mainCode.IndexOf('MeshLinkViewModel.FirewallUnavailableErrorCode', [StringComparison]::Ordinal) -lt 0) {
throw 'Desktop must project both event and persisted Mesh firewall unavailable state.'
}
if ($meshModelsCode.IndexOf('LastError == FirewallUnavailableErrorCode ? string.Empty',
[StringComparison]::Ordinal) -lt 0) {
throw 'Shared Mesh firewall errors must be suppressed from individual Link rows.'
}
if ($mainCode.IndexOf('Servers.Count > 0 || _control.IsConfigured', [StringComparison]::Ordinal) -lt 0 -or
$mainCode.IndexOf('await RefreshControlMeshAsync(showSuccess: false);', [StringComparison]::Ordinal) -lt 0) {
throw 'Configured Control state must refresh even when no SSH profiles exist.'
}
if ($mainCode.IndexOf( if ($mainCode.IndexOf(
'MeshLinksList.SelectedItem = selectedLink;', [StringComparison]::Ordinal) -lt 0) { 'MeshLinksList.SelectedItem = selectedLink;', [StringComparison]::Ordinal) -lt 0) {
throw 'Main page must synchronize the selected Link before disconnecting it.' throw 'Main page must synchronize the selected Link before disconnecting it.'