feat(control): reconcile links from factual state
Add fact-first Link reconciliation, duplicate and orphan cleanup, generation-aware scheduling, retention, Desktop drift visibility, and strict helper contracts. Preserve exact B-3R batching and lock-safe finalization with Linux/native trimmed evidence.
This commit is contained in:
parent
b11c277ac7
commit
ee7d89c1d0
25 changed files with 1901 additions and 167 deletions
|
|
@ -495,6 +495,7 @@ Control__BackupDirectory=$STATE_DIR/backups
|
||||||
Control__HubHelperPath=$POLICY_HELPER
|
Control__HubHelperPath=$POLICY_HELPER
|
||||||
Control__PrivilegeEscalationPath=/usr/bin/sudo
|
Control__PrivilegeEscalationPath=/usr/bin/sudo
|
||||||
Control__LinkReconciliationSeconds=300
|
Control__LinkReconciliationSeconds=300
|
||||||
|
Control__LinkRetentionDays=90
|
||||||
EOF
|
EOF
|
||||||
printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url"
|
printf '%s\n' "https://$public_host:$port" >"$ETC_DIR/control-public-url"
|
||||||
chown root:"$CONTROL_USER" "$ETC_DIR/control.env"
|
chown root:"$CONTROL_USER" "$ETC_DIR/control.env"
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ readonly CHAIN_NAME="links"
|
||||||
|
|
||||||
fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; }
|
fail() { printf '%s\n' "policy helper: $*" >&2; exit 78; }
|
||||||
firewall_unavailable() { printf '%s\n' "mesh.firewall-unavailable" >&2; exit 79; }
|
firewall_unavailable() { printf '%s\n' "mesh.firewall-unavailable" >&2; exit 79; }
|
||||||
|
node_not_activated() { printf '%s\n' "mesh.node-not-activated" >&2; exit 80; }
|
||||||
|
|
||||||
testing="${SMM_POLICY_TESTING:-0}"
|
testing="${SMM_POLICY_TESTING:-0}"
|
||||||
if [[ "$testing" != "1" ]]; then
|
if [[ "$testing" != "1" ]]; then
|
||||||
|
|
@ -28,16 +29,36 @@ fi
|
||||||
node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$'
|
node_pattern='^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$'
|
||||||
ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
|
ipv4_pattern='^([0-9]{1,3}\.){3}[0-9]{1,3}$'
|
||||||
generation_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
generation_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||||
|
# Mesh node lifecycle statuses are lowercase tokens (for example active or reserved).
|
||||||
|
node_status_pattern='^[a-z][a-z0-9-]{0,31}$'
|
||||||
|
|
||||||
validate_node_id() {
|
validate_node_id() {
|
||||||
[[ "$1" =~ $node_pattern ]] || fail "invalid node id"
|
[[ "$1" =~ $node_pattern ]] || fail "invalid node id"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
is_valid_ipv4() {
|
||||||
|
local address="$1" octet
|
||||||
|
local -a octets
|
||||||
|
[[ "$address" =~ $ipv4_pattern ]] || return 1
|
||||||
|
IFS='.' read -r -a octets <<<"$address"
|
||||||
|
[[ ${#octets[@]} -eq 4 ]] || return 1
|
||||||
|
for octet in "${octets[@]}"; do
|
||||||
|
(( 10#$octet <= 255 )) || return 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
lookup_node_ip() {
|
lookup_node_ip() {
|
||||||
local node_id="$1" ip
|
local node_id="$1" record field_count ip status
|
||||||
[[ -r "$STATE_FILE" ]] || fail "mesh node state is unavailable"
|
[[ -r "$STATE_FILE" ]] || fail "mesh node state is unavailable"
|
||||||
ip="$(awk -F '\t' -v node="$node_id" '$1 == node { print $2; exit }' "$STATE_FILE")"
|
record="$(awk -F '\t' -v node="$node_id" '$1 == node { print; exit }' "$STATE_FILE")"
|
||||||
[[ "$ip" =~ $ipv4_pattern ]] || fail "node has no valid mesh address: $node_id"
|
[[ -n "$record" ]] || node_not_activated
|
||||||
|
field_count="$(awk -F '\t' '{ print NF }' <<<"$record")"
|
||||||
|
[[ "$field_count" == "4" ]] || fail "invalid mesh node record: $node_id"
|
||||||
|
ip="$(awk -F '\t' '{ print $2 }' <<<"$record")"
|
||||||
|
status="$(awk -F '\t' '{ print $4 }' <<<"$record")"
|
||||||
|
[[ "$status" =~ $node_status_pattern ]] || fail "invalid mesh node status: $node_id"
|
||||||
|
[[ "$status" == "active" ]] || node_not_activated
|
||||||
|
is_valid_ipv4 "$ip" || fail "node has no valid mesh address: $node_id"
|
||||||
printf '%s\n' "$ip"
|
printf '%s\n' "$ip"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -65,6 +86,7 @@ run_nft() {
|
||||||
printf ' %q' "$@"
|
printf ' %q' "$@"
|
||||||
printf '\n'
|
printf '\n'
|
||||||
else
|
else
|
||||||
|
[[ -x /usr/sbin/nft ]] || fail "nft executable is missing: /usr/sbin/nft"
|
||||||
/usr/sbin/nft "$@"
|
/usr/sbin/nft "$@"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
@ -75,8 +97,13 @@ inspect_firewall() {
|
||||||
[[ "${SMM_POLICY_FIREWALL_UNAVAILABLE:-0}" != "1" ]] || firewall_unavailable
|
[[ "${SMM_POLICY_FIREWALL_UNAVAILABLE:-0}" != "1" ]] || firewall_unavailable
|
||||||
[[ -z "${SMM_POLICY_FIREWALL_ERROR:-}" ]] \
|
[[ -z "${SMM_POLICY_FIREWALL_ERROR:-}" ]] \
|
||||||
|| fail "could not inspect nftables Link policy: $SMM_POLICY_FIREWALL_ERROR"
|
|| fail "could not inspect nftables Link policy: $SMM_POLICY_FIREWALL_ERROR"
|
||||||
|
if [[ -n "${SMM_POLICY_LISTING_FILE:-}" ]]; then
|
||||||
|
[[ -r "$SMM_POLICY_LISTING_FILE" ]] || fail "test firewall listing is unavailable"
|
||||||
|
cat -- "$SMM_POLICY_LISTING_FILE"
|
||||||
|
fi
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
[[ -x /usr/sbin/nft ]] || fail "nft executable is missing: /usr/sbin/nft"
|
||||||
if listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" 2>&1)"; then
|
if listing="$(/usr/sbin/nft -a list chain "$TABLE_FAMILY" "$TABLE_NAME" "$CHAIN_NAME" 2>&1)"; then
|
||||||
printf '%s\n' "$listing"
|
printf '%s\n' "$listing"
|
||||||
return 0
|
return 0
|
||||||
|
|
@ -113,9 +140,6 @@ connect_rule() {
|
||||||
disconnect_rule() {
|
disconnect_rule() {
|
||||||
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle
|
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment handle
|
||||||
ensure_firewall_available
|
ensure_firewall_available
|
||||||
# Resolve both identities before touching firewall state.
|
|
||||||
lookup_node_ip "$source_id" >/dev/null
|
|
||||||
lookup_node_ip "$target_id" >/dev/null
|
|
||||||
comment="smm:${source_id}:${target_id}:${protocol}:${port}"
|
comment="smm:${source_id}:${target_id}:${protocol}:${port}"
|
||||||
if [[ "$testing" == "1" ]]; then
|
if [[ "$testing" == "1" ]]; then
|
||||||
printf 'nft-delete-comment %q\n' "$comment"
|
printf 'nft-delete-comment %q\n' "$comment"
|
||||||
|
|
@ -131,6 +155,39 @@ disconnect_rule() {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
list_rules() {
|
||||||
|
local listing line comment source_id target_id protocol port invalid
|
||||||
|
local -a fields
|
||||||
|
listing="$(inspect_firewall)"
|
||||||
|
while IFS= read -r line; do
|
||||||
|
[[ "$line" =~ comment[[:space:]]+\"([^\"]*)\" ]] || continue
|
||||||
|
comment="${BASH_REMATCH[1]}"
|
||||||
|
[[ "$comment" == smm:* ]] || continue
|
||||||
|
IFS=':' read -r -a fields <<<"$comment"
|
||||||
|
invalid=0
|
||||||
|
if [[ ${#fields[@]} -ne 5 ]]; then
|
||||||
|
invalid=1
|
||||||
|
else
|
||||||
|
source_id="${fields[1]}"
|
||||||
|
target_id="${fields[2]}"
|
||||||
|
protocol="${fields[3]}"
|
||||||
|
port="${fields[4]}"
|
||||||
|
[[ "$source_id" =~ $node_pattern && "$target_id" =~ $node_pattern \
|
||||||
|
&& "$source_id" != "$target_id" ]] || invalid=1
|
||||||
|
[[ "$protocol" == "tcp" || "$protocol" == "udp" ]] || invalid=1
|
||||||
|
if [[ ! "$port" =~ ^[0-9]+$ ]] \
|
||||||
|
|| (( 10#$port < 1 || 10#$port > 65535 )); then
|
||||||
|
invalid=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if (( invalid != 0 )); then
|
||||||
|
printf '%s\n' "policy helper: forged managed comment ignored: $comment" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
printf '%s\t%s\t%s\t%s\n' "$source_id" "$target_id" "$protocol" "$port"
|
||||||
|
done <<<"$listing"
|
||||||
|
}
|
||||||
|
|
||||||
status_rule() {
|
status_rule() {
|
||||||
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment
|
local source_id="$1" target_id="$2" protocol="$3" port="$4" comment
|
||||||
ensure_firewall_available
|
ensure_firewall_available
|
||||||
|
|
@ -147,6 +204,10 @@ status_rule() {
|
||||||
|
|
||||||
reconcile_status() {
|
reconcile_status() {
|
||||||
local generation
|
local generation
|
||||||
|
if [[ ! -d "$(dirname -- "$RECONCILE_MARKER")" ]]; then
|
||||||
|
printf '%s\n' complete
|
||||||
|
return
|
||||||
|
fi
|
||||||
exec 9>"$RECONCILE_LOCK"
|
exec 9>"$RECONCILE_LOCK"
|
||||||
chmod 0600 "$RECONCILE_LOCK"
|
chmod 0600 "$RECONCILE_LOCK"
|
||||||
"$FLOCK_COMMAND" -x 9
|
"$FLOCK_COMMAND" -x 9
|
||||||
|
|
@ -177,6 +238,11 @@ reconcile_complete() {
|
||||||
|
|
||||||
action="${1:-}"
|
action="${1:-}"
|
||||||
case "$action" in
|
case "$action" in
|
||||||
|
link-list)
|
||||||
|
[[ $# -eq 1 ]] || fail "invalid link-list argument count"
|
||||||
|
list_rules
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
reconcile-status)
|
reconcile-status)
|
||||||
[[ $# -eq 1 ]] || fail "invalid reconcile-status argument count"
|
[[ $# -eq 1 ]] || fail "invalid reconcile-status argument count"
|
||||||
reconcile_status
|
reconcile_status
|
||||||
|
|
|
||||||
|
|
@ -108,4 +108,4 @@ sudo ochenstarik-smm-emergency firewall-restore
|
||||||
sudo ochenstarik-smm-emergency mesh-enable
|
sudo ochenstarik-smm-emergency mesh-enable
|
||||||
```
|
```
|
||||||
|
|
||||||
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default и атомарно создаёт root-only запрос реконсиляции с уникальным generation. Control немедленно выполняет первый фоновый проход после старта, затем сверяет все действующие Link-политики не реже настроенного интервала `Control__LinkReconciliationSeconds` (по умолчанию 300 секунд). Новый marker запускает внеочередной проход на следующем poll tick, но не обходит backoff недоступного firewall. Helper удаляет только тот generation, который был прочитан перед успешно завершившимся проходом; более новый запрос сохраняется. При недоступной таблице запрос остаётся для retry с backoff, а Desktop показывает единый баннер «Mesh firewall не загружен». Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` также создаёт запрос реконсиляции; запускайте его только после проверки конфигурации и доступности Hub.
|
`mesh-disable` останавливает WireGuard, удаляет только таблицу `inet ochenstarik_smm` и ставит локальный emergency marker, не останавливая Control, Agent или SSH. `firewall-restore` восстанавливает базовую политику deny-by-default и атомарно создаёт root-only запрос реконсиляции с уникальным generation. Control немедленно выполняет первый фоновый проход после старта, затем одним `link-list` сверяет фактические managed-правила со всеми последними Link-политиками не реже настроенного интервала `Control__LinkReconciliationSeconds` (по умолчанию 300 секунд). Новый marker запускает до трёх внеочередных проходов на следующих poll tick; после трёх отказов обычный throttle восстанавливается, marker сохраняется, а журнал получает одно предупреждение с id проблемных политик. Marker не обходит backoff недоступного firewall. Завершённые политики `Disabled/Disabled` удаляются вместе со своей историей после `Control__LinkRetentionDays` (по умолчанию 90 дней); действующие политики и расхождения retention не затрагивает. Helper удаляет только тот generation, который был прочитан перед успешно завершившимся проходом; более новый запрос сохраняется. При недоступной таблице запрос остаётся для retry с backoff, а Desktop показывает единый баннер «Mesh firewall не загружен». Если firewall не удаётся восстановить, команда отключает Mesh для fail-closed результата. `mesh-enable` также создаёт запрос реконсиляции; запускайте его только после проверки конфигурации и доступности Hub.
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@
|
||||||
- [x] append-only audit операций Link;
|
- [x] append-only audit операций Link;
|
||||||
- [x] интеграционные тесты kill switch, process restart и helper failure;
|
- [x] интеграционные тесты kill switch, process restart и helper failure;
|
||||||
- [x] B-2: независимая фоновая и emergency-triggered реконсиляция, агрегированное состояние недоступного Mesh firewall и Desktop banner;
|
- [x] B-2: независимая фоновая и emergency-triggered реконсиляция, агрегированное состояние недоступного Mesh firewall и Desktop banner;
|
||||||
- [ ] B-3: развести результат реконсиляции на `Examined` / `Converged` / `Failed` (M2), выбрать фильтр или retention завершённых Disabled-политик (M4) и типизировать ещё не активированный Mesh Node (M5). Эти изменения вынесены отдельно, потому что меняют API/UI semantics и helper classification, тогда как B-2 ограничен восстановлением фактических правил и общим firewall failure;
|
- [x] B-3: факт-первичная реконсиляция использует один `link-list`, удаляет orphan/дубликаты (включая `Disabled/Disabled`), разводит `Examined` / `Converged` / `Failed` (M2), ограничивает marker-prompt тремя попытками, добавляет фильтр истории и retention завершённых Links (M4), а также типизированное ожидание активации Mesh Node (M5). Physical acceptance остаётся внешним блокером;
|
||||||
- [ ] выполнить физический acceptance Hub + source Node + два destination Node с WireGuard/nftables/reboot.
|
- [ ] выполнить физический acceptance Hub + source Node + два destination Node с WireGuard/nftables/reboot.
|
||||||
|
|
||||||
## Этап 4 — мониторинг и терминал
|
## Этап 4 — мониторинг и терминал
|
||||||
|
|
|
||||||
13
src/ServerMonitorManager.Control/ControlJsonContext.cs
Normal file
13
src/ServerMonitorManager.Control/ControlJsonContext.cs
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace ServerMonitorManager.Control;
|
||||||
|
|
||||||
|
internal sealed record LinkOrphanAuditDetails(
|
||||||
|
string SourceNodeId,
|
||||||
|
string TargetNodeId,
|
||||||
|
string Protocol,
|
||||||
|
int Port);
|
||||||
|
|
||||||
|
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
|
||||||
|
[JsonSerializable(typeof(LinkOrphanAuditDetails))]
|
||||||
|
internal sealed partial class ControlJsonContext : JsonSerializerContext;
|
||||||
|
|
@ -10,6 +10,7 @@ public sealed record ControlMaintenanceResult(
|
||||||
int MetricsDeleted,
|
int MetricsDeleted,
|
||||||
int IdempotencyDeleted,
|
int IdempotencyDeleted,
|
||||||
int AuditDeleted,
|
int AuditDeleted,
|
||||||
|
int LinksDeleted,
|
||||||
int TokensDeleted,
|
int TokensDeleted,
|
||||||
int ProvisioningJobsCancelled,
|
int ProvisioningJobsCancelled,
|
||||||
int ProvisioningJobsNeedingReconciliation);
|
int ProvisioningJobsNeedingReconciliation);
|
||||||
|
|
@ -68,16 +69,17 @@ public sealed class ControlMaintenanceBackgroundService(
|
||||||
var result = await store.MaintainAsync(timeProvider.GetUtcNow(), stoppingToken);
|
var result = await store.MaintainAsync(timeProvider.GetUtcNow(), stoppingToken);
|
||||||
await backups.CreateIfDueAsync(timeProvider.GetUtcNow(), stoppingToken);
|
await backups.CreateIfDueAsync(timeProvider.GetUtcNow(), stoppingToken);
|
||||||
if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted
|
if (result.MetricsDeleted + result.IdempotencyDeleted + result.AuditDeleted
|
||||||
+ result.TokensDeleted + result.ProvisioningJobsCancelled
|
+ result.LinksDeleted + result.TokensDeleted + result.ProvisioningJobsCancelled
|
||||||
+ result.ProvisioningJobsNeedingReconciliation > 0)
|
+ result.ProvisioningJobsNeedingReconciliation > 0)
|
||||||
{
|
{
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Control maintenance removed {Metrics} metrics, {Idempotency} replay records, "
|
"Control maintenance removed {Metrics} metrics, {Idempotency} replay records, "
|
||||||
+ "{Audit} audit records, and {Tokens} enrollment tokens; cancelled {Cancelled} "
|
+ "{Audit} audit records, {Links} completed Link policies, and {Tokens} enrollment tokens; cancelled {Cancelled} "
|
||||||
+ "expired jobs and marked {Reconciliation} jobs for reconciliation.",
|
+ "expired jobs and marked {Reconciliation} jobs for reconciliation.",
|
||||||
result.MetricsDeleted,
|
result.MetricsDeleted,
|
||||||
result.IdempotencyDeleted,
|
result.IdempotencyDeleted,
|
||||||
result.AuditDeleted,
|
result.AuditDeleted,
|
||||||
|
result.LinksDeleted,
|
||||||
result.TokensDeleted,
|
result.TokensDeleted,
|
||||||
result.ProvisioningJobsCancelled,
|
result.ProvisioningJobsCancelled,
|
||||||
result.ProvisioningJobsNeedingReconciliation);
|
result.ProvisioningJobsNeedingReconciliation);
|
||||||
|
|
|
||||||
|
|
@ -1,38 +1,47 @@
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
|
||||||
namespace ServerMonitorManager.Control;
|
namespace ServerMonitorManager.Control;
|
||||||
|
|
||||||
public sealed class ControlOptions
|
public sealed class ControlOptions
|
||||||
{
|
{
|
||||||
|
[DynamicDependency(DynamicallyAccessedMemberTypes.PublicProperties, typeof(ControlOptions))]
|
||||||
|
public ControlOptions()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
public const string SectionName = "Control";
|
public const string SectionName = "Control";
|
||||||
|
|
||||||
public string DatabasePath { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/control.db";
|
public string DatabasePath { get; set; } = "/var/lib/ochenstarik-server-monitor-manager/control.db";
|
||||||
|
|
||||||
public string CertificateAuthorityPath { get; init; } = "/etc/ochenstarik-server-monitor-manager/control-ca.pfx";
|
public string CertificateAuthorityPath { get; set; } = "/etc/ochenstarik-server-monitor-manager/control-ca.pfx";
|
||||||
|
|
||||||
public string? CertificateAuthorityPassword { get; init; }
|
public string? CertificateAuthorityPassword { get; set; }
|
||||||
|
|
||||||
public int HeartbeatSeconds { get; init; } = 30;
|
public int HeartbeatSeconds { get; set; } = 30;
|
||||||
|
|
||||||
public int MaxBufferedMetricAgeHours { get; init; } = 24;
|
public int MaxBufferedMetricAgeHours { get; set; } = 24;
|
||||||
|
|
||||||
public int MetricRetentionHours { get; init; } = 168;
|
public int MetricRetentionHours { get; set; } = 168;
|
||||||
|
|
||||||
public int IdempotencyRetentionHours { get; init; } = 24;
|
public int IdempotencyRetentionHours { get; set; } = 24;
|
||||||
|
|
||||||
public int AuditRetentionDays { get; init; } = 90;
|
public int AuditRetentionDays { get; set; } = 90;
|
||||||
|
|
||||||
public int MaintenanceIntervalMinutes { get; init; } = 15;
|
public int LinkRetentionDays { get; set; } = 90;
|
||||||
|
|
||||||
public int LinkExpirationPollSeconds { get; init; } = 15;
|
public int MaintenanceIntervalMinutes { get; set; } = 15;
|
||||||
|
|
||||||
public int LinkReconciliationSeconds { get; init; } = 300;
|
public int LinkExpirationPollSeconds { get; set; } = 15;
|
||||||
|
|
||||||
public string BackupDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/backups";
|
public int LinkReconciliationSeconds { get; set; } = 300;
|
||||||
|
|
||||||
public int BackupIntervalHours { get; init; } = 24;
|
public string BackupDirectory { get; set; } = "/var/lib/ochenstarik-server-monitor-manager/backups";
|
||||||
|
|
||||||
public int BackupRetentionCount { get; init; } = 7;
|
public int BackupIntervalHours { get; set; } = 24;
|
||||||
|
|
||||||
public string HubHelperPath { get; init; } = "/usr/local/libexec/ochenstarik-smm-policy-apply";
|
public int BackupRetentionCount { get; set; } = 7;
|
||||||
|
|
||||||
public string PrivilegeEscalationPath { get; init; } = "/usr/bin/sudo";
|
public string HubHelperPath { get; set; } = "/usr/local/libexec/ochenstarik-smm-policy-apply";
|
||||||
|
|
||||||
|
public string PrivilegeEscalationPath { get; set; } = "/usr/bin/sudo";
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -352,6 +352,24 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
SELECT changes();
|
SELECT changes();
|
||||||
DELETE FROM audit WHERE recorded_at < $audit_cutoff;
|
DELETE FROM audit WHERE recorded_at < $audit_cutoff;
|
||||||
SELECT changes();
|
SELECT changes();
|
||||||
|
DELETE FROM links AS historical
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM links AS latest
|
||||||
|
WHERE latest.source_node_id = historical.source_node_id
|
||||||
|
AND latest.target_node_id = historical.target_node_id
|
||||||
|
AND latest.protocol = historical.protocol
|
||||||
|
AND latest.port = historical.port
|
||||||
|
AND latest.desired_state = 'Disabled'
|
||||||
|
AND latest.actual_state = 'Disabled'
|
||||||
|
AND latest.updated_at < $link_cutoff
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM links AS newer
|
||||||
|
WHERE newer.source_node_id = latest.source_node_id
|
||||||
|
AND newer.target_node_id = latest.target_node_id
|
||||||
|
AND newer.protocol = latest.protocol
|
||||||
|
AND newer.port = latest.port
|
||||||
|
AND newer.version > latest.version));
|
||||||
|
SELECT changes();
|
||||||
DELETE FROM enrollment_tokens WHERE consumed_at IS NOT NULL OR expires_at < $now;
|
DELETE FROM enrollment_tokens WHERE consumed_at IS NOT NULL OR expires_at < $now;
|
||||||
SELECT changes();
|
SELECT changes();
|
||||||
DELETE FROM device_tokens WHERE consumed_at IS NOT NULL OR expires_at < $now;
|
DELETE FROM device_tokens WHERE consumed_at IS NOT NULL OR expires_at < $now;
|
||||||
|
|
@ -365,8 +383,10 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
"$idempotency_cutoff", now.AddHours(-_options.IdempotencyRetentionHours).ToString("O"));
|
"$idempotency_cutoff", now.AddHours(-_options.IdempotencyRetentionHours).ToString("O"));
|
||||||
command.Parameters.AddWithValue(
|
command.Parameters.AddWithValue(
|
||||||
"$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O"));
|
"$audit_cutoff", now.AddDays(-_options.AuditRetentionDays).ToString("O"));
|
||||||
|
command.Parameters.AddWithValue(
|
||||||
|
"$link_cutoff", now.AddDays(-_options.LinkRetentionDays).ToString("O"));
|
||||||
command.Parameters.AddWithValue("$now", now.ToString("O"));
|
command.Parameters.AddWithValue("$now", now.ToString("O"));
|
||||||
var changes = new int[9];
|
var changes = new int[10];
|
||||||
await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
|
await using (var reader = await command.ExecuteReaderAsync(cancellationToken))
|
||||||
{
|
{
|
||||||
for (var index = 0; index < changes.Length; index++)
|
for (var index = 0; index < changes.Length; index++)
|
||||||
|
|
@ -384,7 +404,7 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);";
|
optimize.CommandText = "PRAGMA optimize; PRAGMA wal_checkpoint(PASSIVE);";
|
||||||
await optimize.ExecuteNonQueryAsync(cancellationToken);
|
await optimize.ExecuteNonQueryAsync(cancellationToken);
|
||||||
return new ControlMaintenanceResult(
|
return new ControlMaintenanceResult(
|
||||||
changes[3], changes[4], changes[5], changes[6] + changes[7] + changes[8],
|
changes[3], changes[4], changes[5], changes[6], changes[7] + changes[8] + changes[9],
|
||||||
changes[0], changes[1] + changes[2]);
|
changes[0], changes[1] + changes[2]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1353,6 +1373,31 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
return link;
|
return link;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task RecordLinkOrphanRemovedAsync(
|
||||||
|
LinkRule rule,
|
||||||
|
string actor,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken);
|
||||||
|
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||||
|
var subject = $"{rule.SourceNodeId}:{rule.TargetNodeId}:{rule.Protocol}:{rule.Port}";
|
||||||
|
await WriteAuditAsync(
|
||||||
|
connection,
|
||||||
|
transaction,
|
||||||
|
actor,
|
||||||
|
"link.orphan-removed",
|
||||||
|
subject,
|
||||||
|
JsonSerializer.Serialize(
|
||||||
|
new LinkOrphanAuditDetails(
|
||||||
|
rule.SourceNodeId,
|
||||||
|
rule.TargetNodeId,
|
||||||
|
rule.Protocol,
|
||||||
|
rule.Port),
|
||||||
|
ControlJsonContext.Default.LinkOrphanAuditDetails),
|
||||||
|
cancellationToken);
|
||||||
|
await transaction.CommitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<LinkPolicy?> GetLinkAsync(
|
public async Task<LinkPolicy?> GetLinkAsync(
|
||||||
string id,
|
string id,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
|
|
@ -1364,6 +1409,29 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
return link;
|
return link;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<LinkPolicy?> GetEffectiveLinkAsync(
|
||||||
|
LinkRule rule,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken);
|
||||||
|
var command = connection.CreateCommand();
|
||||||
|
command.CommandText = """
|
||||||
|
SELECT * FROM links
|
||||||
|
WHERE source_node_id = $source
|
||||||
|
AND target_node_id = $target
|
||||||
|
AND protocol = $protocol
|
||||||
|
AND port = $port
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1;
|
||||||
|
""";
|
||||||
|
command.Parameters.AddWithValue("$source", rule.SourceNodeId);
|
||||||
|
command.Parameters.AddWithValue("$target", rule.TargetNodeId);
|
||||||
|
command.Parameters.AddWithValue("$protocol", rule.Protocol);
|
||||||
|
command.Parameters.AddWithValue("$port", rule.Port);
|
||||||
|
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||||
|
return await reader.ReadAsync(cancellationToken) ? ReadLink(reader) : null;
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<LinkPolicy>> ListLinksAsync(CancellationToken cancellationToken = default)
|
public async Task<IReadOnlyList<LinkPolicy>> ListLinksAsync(CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var result = new List<LinkPolicy>();
|
var result = new List<LinkPolicy>();
|
||||||
|
|
@ -1387,9 +1455,7 @@ public sealed partial class ControlStore(IOptions<ControlOptions> options)
|
||||||
command.CommandText = """
|
command.CommandText = """
|
||||||
SELECT current.*
|
SELECT current.*
|
||||||
FROM links AS current
|
FROM links AS current
|
||||||
WHERE (current.desired_state = 'Active'
|
WHERE NOT EXISTS (
|
||||||
OR (current.desired_state = 'Disabled' AND current.actual_state != 'Disabled'))
|
|
||||||
AND NOT EXISTS (
|
|
||||||
SELECT 1 FROM links AS newer
|
SELECT 1 FROM links AS newer
|
||||||
WHERE newer.source_node_id = current.source_node_id
|
WHERE newer.source_node_id = current.source_node_id
|
||||||
AND newer.target_node_id = current.target_node_id
|
AND newer.target_node_id = current.target_node_id
|
||||||
|
|
@ -1705,7 +1771,39 @@ public sealed record AgentHeartbeatMutation(
|
||||||
AgentHeartbeatResponse Response,
|
AgentHeartbeatResponse Response,
|
||||||
bool RequiresReconciliation);
|
bool RequiresReconciliation);
|
||||||
|
|
||||||
public sealed record LinkReconciliationResult(int Applied, int Failed);
|
public sealed record LinkReconciliationResult
|
||||||
|
{
|
||||||
|
public LinkReconciliationResult(
|
||||||
|
int examined,
|
||||||
|
int converged,
|
||||||
|
int failed,
|
||||||
|
int deferred,
|
||||||
|
IReadOnlyList<string> failedPolicyIds,
|
||||||
|
IReadOnlyList<string> deferredPolicyIds)
|
||||||
|
{
|
||||||
|
if (converged + failed + deferred != examined)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Link reconciliation classification invariant failed: examined={examined}, "
|
||||||
|
+ $"converged={converged}, failed={failed}, deferred={deferred}; "
|
||||||
|
+ $"failed IDs=[{string.Join(',', failedPolicyIds)}], "
|
||||||
|
+ $"deferred IDs=[{string.Join(',', deferredPolicyIds)}].");
|
||||||
|
}
|
||||||
|
Examined = examined;
|
||||||
|
Converged = converged;
|
||||||
|
Failed = failed;
|
||||||
|
Deferred = deferred;
|
||||||
|
FailedPolicyIds = failedPolicyIds;
|
||||||
|
DeferredPolicyIds = deferredPolicyIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Examined { get; }
|
||||||
|
public int Converged { get; }
|
||||||
|
public int Failed { get; }
|
||||||
|
public int Deferred { get; }
|
||||||
|
public IReadOnlyList<string> FailedPolicyIds { get; }
|
||||||
|
public IReadOnlyList<string> DeferredPolicyIds { get; }
|
||||||
|
}
|
||||||
|
|
||||||
public sealed record AgentReenrollmentMutation(
|
public sealed record AgentReenrollmentMutation(
|
||||||
CertificateReenrollmentTicket Ticket,
|
CertificateReenrollmentTicket Ticket,
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,10 @@ namespace ServerMonitorManager.Control;
|
||||||
|
|
||||||
public interface ILinkPolicyApplier
|
public interface ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
|
Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken);
|
||||||
Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||||
Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||||
|
Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken);
|
||||||
Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken);
|
Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken);
|
||||||
Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
|
Task<string?> GetReconciliationRequestAsync(CancellationToken cancellationToken)
|
||||||
=> Task.FromResult<string?>(null);
|
=> Task.FromResult<string?>(null);
|
||||||
|
|
@ -15,6 +17,8 @@ public interface ILinkPolicyApplier
|
||||||
=> Task.CompletedTask;
|
=> Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed record LinkRule(string SourceNodeId, string TargetNodeId, string Protocol, int Port);
|
||||||
|
|
||||||
public sealed class MeshFirewallUnavailableException : InvalidOperationException
|
public sealed class MeshFirewallUnavailableException : InvalidOperationException
|
||||||
{
|
{
|
||||||
public MeshFirewallUnavailableException(string message) : base(message)
|
public MeshFirewallUnavailableException(string message) : base(message)
|
||||||
|
|
@ -22,8 +26,38 @@ public sealed class MeshFirewallUnavailableException : InvalidOperationException
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class MeshNodeNotActivatedException : InvalidOperationException
|
||||||
|
{
|
||||||
|
public MeshNodeNotActivatedException(string message) : base(message)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkPolicyApplier
|
public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
|
public async Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var output = await RunAsync(["link-list"], cancellationToken);
|
||||||
|
if (output.Length == 0)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
var rules = new List<LinkRule>();
|
||||||
|
foreach (var line in output.Split('\n', StringSplitOptions.RemoveEmptyEntries))
|
||||||
|
{
|
||||||
|
var fields = line.Trim().Split('\t');
|
||||||
|
if (fields.Length != 4
|
||||||
|
|| !int.TryParse(fields[3], System.Globalization.NumberStyles.None,
|
||||||
|
System.Globalization.CultureInfo.InvariantCulture, out var port)
|
||||||
|
|| port is < 1 or > 65535)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Hub policy helper returned an invalid link-list record.");
|
||||||
|
}
|
||||||
|
rules.Add(new LinkRule(fields[0], fields[1], fields[2], port));
|
||||||
|
}
|
||||||
|
return rules;
|
||||||
|
}
|
||||||
|
|
||||||
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
=> RunAsync(
|
=> RunAsync(
|
||||||
[
|
[
|
||||||
|
|
@ -37,13 +71,18 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
|
=> ApplyDisconnectAsync(
|
||||||
|
new LinkRule(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port),
|
||||||
|
cancellationToken);
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
=> RunAsync(
|
=> RunAsync(
|
||||||
[
|
[
|
||||||
"link-disconnect",
|
"link-disconnect",
|
||||||
link.SourceNodeId,
|
rule.SourceNodeId,
|
||||||
link.TargetNodeId,
|
rule.TargetNodeId,
|
||||||
link.Protocol,
|
rule.Protocol,
|
||||||
link.Port.ToString(System.Globalization.CultureInfo.InvariantCulture)
|
rule.Port.ToString(System.Globalization.CultureInfo.InvariantCulture)
|
||||||
],
|
],
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
|
|
@ -114,6 +153,11 @@ public sealed class LinkPolicyApplier(IOptions<ControlOptions> options) : ILinkP
|
||||||
{
|
{
|
||||||
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
||||||
}
|
}
|
||||||
|
if (process.ExitCode == 80
|
||||||
|
&& string.Equals(message, LinkService.NodeNotActivatedCode, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
throw new MeshNodeNotActivatedException(LinkService.NodeNotActivatedCode);
|
||||||
|
}
|
||||||
throw new InvalidOperationException(string.IsNullOrWhiteSpace(message)
|
throw new InvalidOperationException(string.IsNullOrWhiteSpace(message)
|
||||||
? $"Hub policy helper exited with code {process.ExitCode}."
|
? $"Hub policy helper exited with code {process.ExitCode}."
|
||||||
: message);
|
: message);
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,13 @@ public sealed class LinkReconciliationBackgroundService(
|
||||||
TimeProvider timeProvider,
|
TimeProvider timeProvider,
|
||||||
ILogger<LinkReconciliationBackgroundService> logger) : BackgroundService
|
ILogger<LinkReconciliationBackgroundService> logger) : BackgroundService
|
||||||
{
|
{
|
||||||
|
private const int PromptAttemptLimit = 3;
|
||||||
private readonly SemaphoreSlim _passGate = new(1, 1);
|
private readonly SemaphoreSlim _passGate = new(1, 1);
|
||||||
private DateTimeOffset? _nextRegularAt;
|
private DateTimeOffset? _nextRegularAt;
|
||||||
private DateTimeOffset? _backoffUntil;
|
private DateTimeOffset? _backoffUntil;
|
||||||
private int _unavailableAttempts;
|
private int _unavailableAttempts;
|
||||||
|
private int _promptFailureAttempts;
|
||||||
|
private bool _promptThrottleWarningLogged;
|
||||||
|
|
||||||
internal async Task<LinkFullReconciliationResult?> RunOnceAsync(
|
internal async Task<LinkFullReconciliationResult?> RunOnceAsync(
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
|
|
@ -34,27 +37,59 @@ public sealed class LinkReconciliationBackgroundService(
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (requestGeneration is null && _nextRegularAt is not null && now < _nextRegularAt)
|
var promptBypassesThrottle = requestGeneration is not null
|
||||||
|
&& _promptFailureAttempts < PromptAttemptLimit;
|
||||||
|
if (!promptBypassesThrottle && _nextRegularAt is not null && now < _nextRegularAt)
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
var result = await links.ReconcileAllAsync(cancellationToken);
|
|
||||||
var interval = TimeSpan.FromSeconds(options.Value.LinkReconciliationSeconds);
|
var interval = TimeSpan.FromSeconds(options.Value.LinkReconciliationSeconds);
|
||||||
|
_nextRegularAt = now + interval;
|
||||||
|
LinkFullReconciliationResult result;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
result = await links.ReconcileAllAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (
|
||||||
|
exception is not OperationCanceledException && requestGeneration is not null)
|
||||||
|
{
|
||||||
|
RegisterPromptFailure([]);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
if (result.FirewallUnavailable)
|
if (result.FirewallUnavailable)
|
||||||
{
|
{
|
||||||
_unavailableAttempts = Math.Min(_unavailableAttempts + 1, 4);
|
_unavailableAttempts = Math.Min(_unavailableAttempts + 1, 4);
|
||||||
_backoffUntil = now + TimeSpan.FromTicks(interval.Ticks * _unavailableAttempts);
|
_backoffUntil = now + TimeSpan.FromTicks(interval.Ticks * _unavailableAttempts);
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
_unavailableAttempts = 0;
|
||||||
|
_backoffUntil = null;
|
||||||
|
if (requestGeneration is null)
|
||||||
{
|
{
|
||||||
_unavailableAttempts = 0;
|
return result;
|
||||||
_backoffUntil = null;
|
}
|
||||||
_nextRegularAt = now + interval;
|
|
||||||
if (requestGeneration is not null && result.Failed == 0)
|
if (result.Failed > 0)
|
||||||
{
|
{
|
||||||
await applier.CompleteReconciliationAsync(requestGeneration, cancellationToken);
|
RegisterPromptFailure(result.FailedPolicyIds);
|
||||||
}
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await applier.CompleteReconciliationAsync(requestGeneration, cancellationToken);
|
||||||
|
_promptFailureAttempts = 0;
|
||||||
|
_promptThrottleWarningLogged = false;
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Link reconciliation completed, but marker generation {Generation} could not be consumed.",
|
||||||
|
requestGeneration);
|
||||||
|
RegisterPromptFailure(["marker-completion"]);
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
@ -64,6 +99,19 @@ public sealed class LinkReconciliationBackgroundService(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void RegisterPromptFailure(IReadOnlyList<string> failedPolicyIds)
|
||||||
|
{
|
||||||
|
_promptFailureAttempts = Math.Min(_promptFailureAttempts + 1, PromptAttemptLimit);
|
||||||
|
if (_promptFailureAttempts == PromptAttemptLimit && !_promptThrottleWarningLogged)
|
||||||
|
{
|
||||||
|
_promptThrottleWarningLogged = true;
|
||||||
|
logger.LogWarning(
|
||||||
|
"Link reconciliation marker prompt exhausted after {Attempts} failures; regular throttle restored. Failed policy IDs: {FailedPolicyIds}.",
|
||||||
|
PromptAttemptLimit,
|
||||||
|
failedPolicyIds.Count == 0 ? "unknown" : string.Join(",", failedPolicyIds));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
var pollSeconds = Math.Min(options.Value.LinkReconciliationSeconds, 30);
|
var pollSeconds = Math.Min(options.Value.LinkReconciliationSeconds, 30);
|
||||||
|
|
@ -76,10 +124,15 @@ public sealed class LinkReconciliationBackgroundService(
|
||||||
if (result is not null)
|
if (result is not null)
|
||||||
{
|
{
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Link reconciliation completed: {Examined} examined, {Failed} failed, firewall unavailable: {Unavailable}.",
|
"Link reconciliation completed: {Examined} examined, {Converged} converged, {Deferred} deferred, {Failed} failed, firewall unavailable: {Unavailable}. Deferred policy IDs: {DeferredPolicyIds}.",
|
||||||
result.Examined,
|
result.Examined,
|
||||||
|
result.Converged,
|
||||||
|
result.Deferred,
|
||||||
result.Failed,
|
result.Failed,
|
||||||
result.FirewallUnavailable);
|
result.FirewallUnavailable,
|
||||||
|
result.DeferredPolicyIds.Count == 0
|
||||||
|
? "none"
|
||||||
|
: string.Join(",", result.DeferredPolicyIds));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ public sealed class LinkService(
|
||||||
{
|
{
|
||||||
public const string FirewallUnavailableCode = "mesh.firewall-unavailable";
|
public const string FirewallUnavailableCode = "mesh.firewall-unavailable";
|
||||||
public const string FirewallAvailableCode = "mesh.firewall-available";
|
public const string FirewallAvailableCode = "mesh.firewall-available";
|
||||||
|
public const string NodeNotActivatedCode = "mesh.node-not-activated";
|
||||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _reconciliationLocks = new();
|
private readonly ConcurrentDictionary<string, SemaphoreSlim> _reconciliationLocks = new();
|
||||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _nodeLocks = new();
|
private readonly ConcurrentDictionary<string, SemaphoreSlim> _nodeLocks = new();
|
||||||
|
|
||||||
|
|
@ -108,8 +109,12 @@ public sealed class LinkService(
|
||||||
string nodeId,
|
string nodeId,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var reconciled = 0;
|
var examined = 0;
|
||||||
|
var converged = 0;
|
||||||
|
var deferred = 0;
|
||||||
var failed = 0;
|
var failed = 0;
|
||||||
|
var failedPolicyIds = new List<string>();
|
||||||
|
var deferredPolicyIds = new List<string>();
|
||||||
var links = await store.ListEffectiveLinksForNodeAsync(nodeId, cancellationToken);
|
var links = await store.ListEffectiveLinksForNodeAsync(nodeId, cancellationToken);
|
||||||
foreach (var candidate in links)
|
foreach (var candidate in links)
|
||||||
{
|
{
|
||||||
|
|
@ -128,10 +133,20 @@ public sealed class LinkService(
|
||||||
current.DesiredState == "Active",
|
current.DesiredState == "Active",
|
||||||
$"system:reconnect:{nodeId}",
|
$"system:reconnect:{nodeId}",
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
reconciled++;
|
examined++;
|
||||||
if (result.ActualState is "Failed" or "Partial")
|
if (result.ActualState is "Failed" or "Partial")
|
||||||
{
|
{
|
||||||
failed++;
|
failed++;
|
||||||
|
failedPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else if (result.ActualState == "PendingActivation")
|
||||||
|
{
|
||||||
|
deferred++;
|
||||||
|
deferredPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else if (result.ActualState == (current.DesiredState == "Active" ? "Active" : "Disabled"))
|
||||||
|
{
|
||||||
|
converged++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
|
|
@ -139,61 +154,223 @@ public sealed class LinkService(
|
||||||
gate.Release();
|
gate.Release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return new LinkReconciliationResult(reconciled, failed);
|
return new LinkReconciliationResult(
|
||||||
|
examined, converged, failed, deferred, failedPolicyIds, deferredPolicyIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<LinkFullReconciliationResult> ReconcileAllAsync(
|
public async Task<LinkFullReconciliationResult> ReconcileAllAsync(
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
|
IReadOnlyList<LinkRule> factualRules;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
factualRules = await applier.ListRulesAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
catch (MeshFirewallUnavailableException)
|
||||||
|
{
|
||||||
|
var unavailableCandidates = await store.ListEffectiveLinksAsync(cancellationToken);
|
||||||
|
return await CompleteFirewallUnavailablePassAsync(
|
||||||
|
unavailableCandidates, 0, 0, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
var candidates = await store.ListEffectiveLinksAsync(cancellationToken);
|
var candidates = await store.ListEffectiveLinksAsync(cancellationToken);
|
||||||
var recoveringFirewall = candidates.Any(candidate =>
|
var recoveringFirewall = candidates.Any(candidate =>
|
||||||
string.Equals(candidate.LastError, FirewallUnavailableCode, StringComparison.Ordinal));
|
string.Equals(candidate.LastError, FirewallUnavailableCode, StringComparison.Ordinal));
|
||||||
|
var factualCounts = factualRules
|
||||||
|
.GroupBy(static rule => rule)
|
||||||
|
.ToDictionary(static group => group.Key, static group => group.Count());
|
||||||
|
var processedKeys = new HashSet<LinkRule>();
|
||||||
|
var batch = new FullReconciliationBatch();
|
||||||
|
var pendingClassifications = new List<(string Id, string ExpectedState)>();
|
||||||
var examined = 0;
|
var examined = 0;
|
||||||
|
var converged = 0;
|
||||||
|
var deferred = 0;
|
||||||
var failed = 0;
|
var failed = 0;
|
||||||
var firewallUnavailable = false;
|
var failedPolicyIds = new List<string>();
|
||||||
|
var deferredPolicyIds = new List<string>();
|
||||||
|
|
||||||
foreach (var candidate in candidates)
|
foreach (var candidate in candidates)
|
||||||
{
|
{
|
||||||
using var nodeLease = await AcquireNodeLocksAsync(
|
var firewallUnavailable = false;
|
||||||
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
using (await AcquireNodeLocksAsync(
|
||||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken))
|
||||||
try
|
|
||||||
{
|
{
|
||||||
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
|
var candidateRule = ToRule(candidate);
|
||||||
if (current is null || !IsEligibleForFullReconciliation(current))
|
var selected = await store.GetEffectiveLinkAsync(candidateRule, cancellationToken);
|
||||||
|
if (selected is null)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
var result = await ConvergeAsync(
|
var gate = await AcquireLinkGateAsync(selected.Id, cancellationToken);
|
||||||
current, current.DesiredState == "Active", "system:reconcile", cancellationToken);
|
try
|
||||||
examined++;
|
|
||||||
if (result.LastError == FirewallUnavailableCode)
|
|
||||||
{
|
{
|
||||||
firewallUnavailable = true;
|
var current = await store.GetEffectiveLinkAsync(candidateRule, cancellationToken);
|
||||||
|
if (current is null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var rule = ToRule(current);
|
||||||
|
if (!processedKeys.Add(rule))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
factualCounts.TryGetValue(rule, out var factualCount);
|
||||||
|
var result = await ConvergeAsync(
|
||||||
|
current,
|
||||||
|
current.DesiredState == "Active",
|
||||||
|
"system:reconcile",
|
||||||
|
cancellationToken,
|
||||||
|
factualCount,
|
||||||
|
batch: batch);
|
||||||
|
examined++;
|
||||||
|
if (result.LastError == FirewallUnavailableCode)
|
||||||
|
{
|
||||||
|
firewallUnavailable = true;
|
||||||
|
}
|
||||||
|
else if (result.ActualState is "Failed" or "Partial")
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
failedPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else if (result.ActualState == "PendingActivation")
|
||||||
|
{
|
||||||
|
deferred++;
|
||||||
|
deferredPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else if (result.ActualState == (current.DesiredState == "Active" ? "Active" : "Disabled"))
|
||||||
|
{
|
||||||
|
converged++;
|
||||||
|
}
|
||||||
|
else if (batch.Contains(result.Id))
|
||||||
|
{
|
||||||
|
pendingClassifications.Add((
|
||||||
|
result.Id,
|
||||||
|
current.DesiredState == "Active" ? "Active" : "Disabled"));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
failedPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (result.ActualState is "Failed" or "Partial")
|
finally
|
||||||
{
|
{
|
||||||
failed++;
|
gate.Release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
|
||||||
{
|
|
||||||
gate.Release();
|
|
||||||
}
|
|
||||||
if (firewallUnavailable)
|
if (firewallUnavailable)
|
||||||
{
|
{
|
||||||
break;
|
return await CompleteFirewallUnavailablePassAsync(
|
||||||
|
candidates, examined, converged, cancellationToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (firewallUnavailable)
|
|
||||||
|
foreach (var orphan in factualCounts.Keys.Where(rule => !processedKeys.Contains(rule)))
|
||||||
{
|
{
|
||||||
await MarkFirewallUnavailableAsync(candidates, cancellationToken);
|
var firewallUnavailable = false;
|
||||||
events.Publish(
|
using (await AcquireNodeLocksAsync(
|
||||||
FirewallUnavailableCode,
|
[orphan.SourceNodeId, orphan.TargetNodeId], cancellationToken))
|
||||||
"mesh",
|
{
|
||||||
JsonSerializer.Serialize(
|
var selected = await store.GetEffectiveLinkAsync(orphan, cancellationToken);
|
||||||
new ControlError(FirewallUnavailableCode), SmmJsonContext.Default.ControlError));
|
var gate = await AcquireLinkGateAsync(
|
||||||
return new LinkFullReconciliationResult(examined, candidates.Count, true);
|
selected?.Id
|
||||||
|
?? $"orphan:{orphan.SourceNodeId}:{orphan.TargetNodeId}:{orphan.Protocol}:{orphan.Port}",
|
||||||
|
cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var persisted = await store.GetEffectiveLinkAsync(orphan, cancellationToken);
|
||||||
|
var target = persisted ?? new LinkPolicy(
|
||||||
|
$"orphan:{orphan.SourceNodeId}:{orphan.TargetNodeId}:{orphan.Protocol}:{orphan.Port}",
|
||||||
|
orphan.SourceNodeId,
|
||||||
|
orphan.TargetNodeId,
|
||||||
|
orphan.Protocol,
|
||||||
|
orphan.Port,
|
||||||
|
0,
|
||||||
|
"factual orphan",
|
||||||
|
"Disabled",
|
||||||
|
"Active",
|
||||||
|
0,
|
||||||
|
DateTimeOffset.UtcNow,
|
||||||
|
null,
|
||||||
|
DateTimeOffset.UtcNow,
|
||||||
|
null);
|
||||||
|
var result = await ConvergeAsync(
|
||||||
|
target,
|
||||||
|
expectedConnected: persisted?.DesiredState == "Active",
|
||||||
|
"system:reconcile",
|
||||||
|
cancellationToken,
|
||||||
|
factualCounts[orphan],
|
||||||
|
persisted: persisted is not null,
|
||||||
|
batch: batch);
|
||||||
|
examined++;
|
||||||
|
var expectedState = persisted?.DesiredState == "Active" ? "Active" : "Disabled";
|
||||||
|
if (result.LastError == FirewallUnavailableCode)
|
||||||
|
{
|
||||||
|
firewallUnavailable = true;
|
||||||
|
}
|
||||||
|
else if (result.ActualState == expectedState)
|
||||||
|
{
|
||||||
|
converged++;
|
||||||
|
}
|
||||||
|
else if (result.ActualState == "PendingActivation")
|
||||||
|
{
|
||||||
|
deferred++;
|
||||||
|
deferredPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else if (batch.Contains(result.Id))
|
||||||
|
{
|
||||||
|
pendingClassifications.Add((result.Id, expectedState));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
failedPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (firewallUnavailable)
|
||||||
|
{
|
||||||
|
return await CompleteFirewallUnavailablePassAsync(
|
||||||
|
candidates, examined, converged, cancellationToken);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (batch.MutationAttempted)
|
||||||
|
{
|
||||||
|
IReadOnlyDictionary<string, LinkPolicy> finalized;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
finalized = await FinalizeBatchAsync(batch, cancellationToken);
|
||||||
|
}
|
||||||
|
catch (MeshFirewallUnavailableException)
|
||||||
|
{
|
||||||
|
return await CompleteFirewallUnavailablePassAsync(
|
||||||
|
candidates, examined, converged, cancellationToken);
|
||||||
|
}
|
||||||
|
foreach (var pendingClassification in pendingClassifications)
|
||||||
|
{
|
||||||
|
var result = finalized[pendingClassification.Id];
|
||||||
|
if (result.ActualState == pendingClassification.ExpectedState)
|
||||||
|
{
|
||||||
|
converged++;
|
||||||
|
}
|
||||||
|
else if (result.ActualState == "PendingActivation")
|
||||||
|
{
|
||||||
|
deferred++;
|
||||||
|
deferredPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
failedPolicyIds.Add(result.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (recoveringFirewall)
|
if (recoveringFirewall)
|
||||||
{
|
{
|
||||||
events.Publish(
|
events.Publish(
|
||||||
|
|
@ -202,7 +379,14 @@ public sealed class LinkService(
|
||||||
JsonSerializer.Serialize(
|
JsonSerializer.Serialize(
|
||||||
new ControlError(FirewallAvailableCode), SmmJsonContext.Default.ControlError));
|
new ControlError(FirewallAvailableCode), SmmJsonContext.Default.ControlError));
|
||||||
}
|
}
|
||||||
return new LinkFullReconciliationResult(examined, failed, false);
|
return new LinkFullReconciliationResult(
|
||||||
|
examined,
|
||||||
|
converged,
|
||||||
|
failed,
|
||||||
|
deferred,
|
||||||
|
false,
|
||||||
|
failedPolicyIds,
|
||||||
|
deferredPolicyIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<LinkExpirationResult> ExpireDueLinksAsync(
|
public async Task<LinkExpirationResult> ExpireDueLinksAsync(
|
||||||
|
|
@ -259,11 +443,17 @@ public sealed class LinkService(
|
||||||
LinkPolicy link,
|
LinkPolicy link,
|
||||||
bool expectedConnected,
|
bool expectedConnected,
|
||||||
string actor,
|
string actor,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken,
|
||||||
|
int? knownFactualCount = null,
|
||||||
|
bool persisted = true,
|
||||||
|
FullReconciliationBatch? batch = null)
|
||||||
{
|
{
|
||||||
var current = await store.GetLinkAsync(link.Id, cancellationToken) ?? link;
|
var current = persisted
|
||||||
if ((current.DesiredState == "Active") != expectedConnected
|
? await store.GetLinkAsync(link.Id, cancellationToken) ?? link
|
||||||
|| !await store.IsEffectiveLinkAsync(current, cancellationToken))
|
: link;
|
||||||
|
if (persisted
|
||||||
|
&& ((current.DesiredState == "Active") != expectedConnected
|
||||||
|
|| !await store.IsEffectiveLinkAsync(current, cancellationToken)))
|
||||||
{
|
{
|
||||||
return current;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
@ -274,45 +464,87 @@ public sealed class LinkService(
|
||||||
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
|
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var isConnected = await applier.IsConnectedAsync(current, cancellationToken);
|
var factualCount = knownFactualCount
|
||||||
var changedFact = isConnected != expectedConnected;
|
?? await CountExactRulesAsync(current, cancellationToken);
|
||||||
|
var isConnected = factualCount > 0;
|
||||||
|
var duplicateActiveRule = expectedConnected && factualCount > 1;
|
||||||
|
var changedFact = isConnected != expectedConnected || duplicateActiveRule;
|
||||||
if (changedFact)
|
if (changedFact)
|
||||||
{
|
{
|
||||||
var pendingState = expectedConnected ? "Connecting" : "Disconnecting";
|
var pendingState = expectedConnected ? "Connecting" : "Disconnecting";
|
||||||
if (current.ActualState != pendingState || current.LastError is not null)
|
if (persisted && (current.ActualState != pendingState || current.LastError is not null))
|
||||||
{
|
{
|
||||||
current = await store.SetLinkActualStateAsync(
|
current = await store.SetLinkActualStateAsync(
|
||||||
current.Id, pendingState, null, actor, cancellationToken) ?? current;
|
current.Id, pendingState, null, actor, cancellationToken) ?? current;
|
||||||
}
|
}
|
||||||
|
if (duplicateActiveRule)
|
||||||
|
{
|
||||||
|
batch?.MarkMutationAttempted();
|
||||||
|
await applier.ApplyDisconnectAsync(current, cancellationToken);
|
||||||
|
if (batch is null)
|
||||||
|
{
|
||||||
|
await VerifyExactFactualCountAsync(current, 0, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (expectedConnected)
|
||||||
|
{
|
||||||
|
batch?.MarkMutationAttempted();
|
||||||
|
await applier.ApplyConnectAsync(current, cancellationToken);
|
||||||
|
if (batch is null)
|
||||||
|
{
|
||||||
|
await VerifyExactFactualCountAsync(current, 1, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (persisted)
|
||||||
|
{
|
||||||
|
batch?.MarkMutationAttempted();
|
||||||
|
await applier.ApplyDisconnectAsync(current, cancellationToken);
|
||||||
|
if (batch is null)
|
||||||
|
{
|
||||||
|
await VerifyExactFactualCountAsync(current, 0, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
batch?.MarkMutationAttempted();
|
||||||
|
await applier.ApplyDisconnectAsync(ToRule(current), cancellationToken);
|
||||||
|
if (batch is null)
|
||||||
|
{
|
||||||
|
await VerifyExactFactualCountAsync(current, 0, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (actor.StartsWith("system:", StringComparison.Ordinal))
|
if (actor.StartsWith("system:", StringComparison.Ordinal))
|
||||||
{
|
{
|
||||||
Publish("link.reconciling", current);
|
Publish("link.reconciling", current);
|
||||||
}
|
}
|
||||||
if (expectedConnected)
|
if (batch is not null)
|
||||||
{
|
{
|
||||||
await applier.ApplyConnectAsync(current, cancellationToken);
|
batch.Stage(current, expectedConnected, actor, persisted);
|
||||||
|
return current;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
await applier.ApplyDisconnectAsync(current, cancellationToken);
|
|
||||||
}
|
|
||||||
await VerifyFactualStateAsync(current, expectedConnected, cancellationToken);
|
|
||||||
}
|
}
|
||||||
if (current.ActualState != completedState || current.LastError is not null)
|
current = await FinalizeConvergenceAsync(
|
||||||
|
current, expectedConnected, actor, persisted, changedFact, cancellationToken);
|
||||||
|
}
|
||||||
|
catch (MeshNodeNotActivatedException) when (expectedConnected)
|
||||||
|
{
|
||||||
|
if (persisted)
|
||||||
{
|
{
|
||||||
current = await store.SetLinkActualStateAsync(
|
current = await store.SetLinkActualStateAsync(
|
||||||
current.Id, completedState, null, actor, cancellationToken) ?? current;
|
current.Id, "PendingActivation", NodeNotActivatedCode, actor, cancellationToken) ?? current;
|
||||||
Publish(completedEvent, current);
|
Publish("link.pending-node-activation", current);
|
||||||
}
|
}
|
||||||
if (changedFact && actor == "system:reconcile")
|
else
|
||||||
{
|
{
|
||||||
Publish(expectedConnected ? "link.reapplied" : "link.orphan-removed", current);
|
current = current with { ActualState = "PendingActivation", LastError = NodeNotActivatedCode };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (MeshFirewallUnavailableException)
|
catch (MeshFirewallUnavailableException)
|
||||||
{
|
{
|
||||||
current = await store.SetLinkActualStateAsync(
|
current = persisted
|
||||||
current.Id, failureState, FirewallUnavailableCode, actor, cancellationToken) ?? current;
|
? await store.SetLinkActualStateAsync(
|
||||||
|
current.Id, failureState, FirewallUnavailableCode, actor, cancellationToken) ?? current
|
||||||
|
: current with { ActualState = failureState, LastError = FirewallUnavailableCode };
|
||||||
if (actor != "system:reconcile")
|
if (actor != "system:reconcile")
|
||||||
{
|
{
|
||||||
Publish(failureEvent, current);
|
Publish(failureEvent, current);
|
||||||
|
|
@ -320,13 +552,148 @@ public sealed class LinkService(
|
||||||
}
|
}
|
||||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
{
|
{
|
||||||
current = await store.SetLinkActualStateAsync(
|
var error = CompactError(exception);
|
||||||
current.Id, failureState, CompactError(exception), actor, cancellationToken) ?? current;
|
current = persisted
|
||||||
|
? await store.SetLinkActualStateAsync(
|
||||||
|
current.Id, failureState, error, actor, cancellationToken) ?? current
|
||||||
|
: current with { ActualState = failureState, LastError = error };
|
||||||
Publish(failureEvent, current);
|
Publish(failureEvent, current);
|
||||||
}
|
}
|
||||||
return current;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyDictionary<string, LinkPolicy>> FinalizeBatchAsync(
|
||||||
|
FullReconciliationBatch batch,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var pendingItems = batch.Pending.ToArray();
|
||||||
|
using var nodeLease = await AcquireNodeLocksAsync(
|
||||||
|
pendingItems.SelectMany(static pending =>
|
||||||
|
new[] { pending.Link.SourceNodeId, pending.Link.TargetNodeId }),
|
||||||
|
cancellationToken);
|
||||||
|
var selectedByPendingId = new Dictionary<string, LinkPolicy?>(StringComparer.Ordinal);
|
||||||
|
foreach (var pending in pendingItems)
|
||||||
|
{
|
||||||
|
selectedByPendingId[pending.Link.Id] = await store.GetEffectiveLinkAsync(
|
||||||
|
ToRule(pending.Link), cancellationToken);
|
||||||
|
}
|
||||||
|
using var linkLease = await AcquireLinkGatesAsync(
|
||||||
|
pendingItems.Select(pending =>
|
||||||
|
selectedByPendingId[pending.Link.Id]?.Id ?? pending.Link.Id),
|
||||||
|
cancellationToken);
|
||||||
|
var finalRules = await applier.ListRulesAsync(cancellationToken);
|
||||||
|
var currentByPendingId = new Dictionary<string, LinkPolicy?>(StringComparer.Ordinal);
|
||||||
|
foreach (var pending in pendingItems)
|
||||||
|
{
|
||||||
|
currentByPendingId[pending.Link.Id] = await store.GetEffectiveLinkAsync(
|
||||||
|
ToRule(pending.Link), cancellationToken);
|
||||||
|
}
|
||||||
|
var finalCounts = finalRules
|
||||||
|
.GroupBy(static rule => rule)
|
||||||
|
.ToDictionary(static group => group.Key, static group => group.Count());
|
||||||
|
var finalized = new Dictionary<string, LinkPolicy>(StringComparer.Ordinal);
|
||||||
|
foreach (var pending in pendingItems)
|
||||||
|
{
|
||||||
|
var current = currentByPendingId[pending.Link.Id];
|
||||||
|
if (IsStale(pending, current))
|
||||||
|
{
|
||||||
|
finalized.Add(pending.Link.Id, CreateStaleBatchResult(pending, current));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
finalCounts.TryGetValue(ToRule(pending.Link), out var actualCount);
|
||||||
|
var expectedCount = pending.ExpectedConnected ? 1 : 0;
|
||||||
|
LinkPolicy result;
|
||||||
|
if (actualCount == expectedCount)
|
||||||
|
{
|
||||||
|
result = await FinalizeConvergenceAsync(
|
||||||
|
pending.Link,
|
||||||
|
pending.ExpectedConnected,
|
||||||
|
pending.Actor,
|
||||||
|
pending.Persisted,
|
||||||
|
changedFact: true,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result = await FailConvergenceAsync(
|
||||||
|
pending.Link,
|
||||||
|
pending.ExpectedConnected,
|
||||||
|
pending.Actor,
|
||||||
|
pending.Persisted,
|
||||||
|
$"Factual Link policy count must be exactly {expectedCount} after application, but was {actualCount}.",
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
finalized.Add(pending.Link.Id, result);
|
||||||
|
}
|
||||||
|
return finalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsStale(PendingConvergence pending, LinkPolicy? current)
|
||||||
|
=> pending.Persisted
|
||||||
|
? current is null
|
||||||
|
|| current.Id != pending.Link.Id
|
||||||
|
|| current.Version != pending.Link.Version
|
||||||
|
|| (current.DesiredState == "Active") != pending.ExpectedConnected
|
||||||
|
: current is not null;
|
||||||
|
|
||||||
|
private static LinkPolicy CreateStaleBatchResult(
|
||||||
|
PendingConvergence pending,
|
||||||
|
LinkPolicy? current)
|
||||||
|
=> (current ?? pending.Link) with
|
||||||
|
{
|
||||||
|
ActualState = pending.ExpectedConnected ? "Failed" : "Partial",
|
||||||
|
LastError = "Link policy changed before batch finalization."
|
||||||
|
};
|
||||||
|
|
||||||
|
private async Task<LinkPolicy> FinalizeConvergenceAsync(
|
||||||
|
LinkPolicy current,
|
||||||
|
bool expectedConnected,
|
||||||
|
string actor,
|
||||||
|
bool persisted,
|
||||||
|
bool changedFact,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var completedState = expectedConnected ? "Active" : "Disabled";
|
||||||
|
var completedEvent = expectedConnected ? "link.active" : "link.disabled";
|
||||||
|
if (persisted && (current.ActualState != completedState || current.LastError is not null))
|
||||||
|
{
|
||||||
|
current = await store.SetLinkActualStateAsync(
|
||||||
|
current.Id, completedState, null, actor, cancellationToken) ?? current;
|
||||||
|
Publish(completedEvent, current);
|
||||||
|
}
|
||||||
|
else if (!persisted)
|
||||||
|
{
|
||||||
|
current = current with { ActualState = completedState, LastError = null };
|
||||||
|
}
|
||||||
|
if (changedFact && actor == "system:reconcile")
|
||||||
|
{
|
||||||
|
if (!expectedConnected)
|
||||||
|
{
|
||||||
|
await store.RecordLinkOrphanRemovedAsync(ToRule(current), actor, cancellationToken);
|
||||||
|
}
|
||||||
|
Publish(expectedConnected ? "link.reapplied" : "link.orphan-removed", current);
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<LinkPolicy> FailConvergenceAsync(
|
||||||
|
LinkPolicy current,
|
||||||
|
bool expectedConnected,
|
||||||
|
string actor,
|
||||||
|
bool persisted,
|
||||||
|
string error,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var failureState = expectedConnected ? "Failed" : "Partial";
|
||||||
|
var failureEvent = expectedConnected ? "link.failed" : "link.partial";
|
||||||
|
current = persisted
|
||||||
|
? await store.SetLinkActualStateAsync(
|
||||||
|
current.Id, failureState, error, actor, cancellationToken) ?? current
|
||||||
|
: current with { ActualState = failureState, LastError = error };
|
||||||
|
Publish(failureEvent, current);
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
private async Task<LinkPolicy?> GetCurrentEffectiveAsync(
|
private async Task<LinkPolicy?> GetCurrentEffectiveAsync(
|
||||||
string id,
|
string id,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
|
|
@ -347,11 +714,17 @@ public sealed class LinkService(
|
||||||
{
|
{
|
||||||
using var nodeLease = await AcquireNodeLocksAsync(
|
using var nodeLease = await AcquireNodeLocksAsync(
|
||||||
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
[candidate.SourceNodeId, candidate.TargetNodeId], cancellationToken);
|
||||||
var gate = await AcquireLinkGateAsync(candidate.Id, cancellationToken);
|
var rule = ToRule(candidate);
|
||||||
|
var selected = await store.GetEffectiveLinkAsync(rule, cancellationToken);
|
||||||
|
if (selected is null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var gate = await AcquireLinkGateAsync(selected.Id, cancellationToken);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var current = await GetCurrentEffectiveAsync(candidate.Id, cancellationToken);
|
var current = await store.GetEffectiveLinkAsync(rule, cancellationToken);
|
||||||
if (current is not null && IsEligibleForFullReconciliation(current))
|
if (current is not null)
|
||||||
{
|
{
|
||||||
await store.SetLinkActualStateAsync(
|
await store.SetLinkActualStateAsync(
|
||||||
current.Id, "Partial", FirewallUnavailableCode, "system:reconcile", cancellationToken);
|
current.Id, "Partial", FirewallUnavailableCode, "system:reconcile", cancellationToken);
|
||||||
|
|
@ -364,9 +737,29 @@ public sealed class LinkService(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool IsEligibleForFullReconciliation(LinkPolicy link)
|
private async Task<LinkFullReconciliationResult> CompleteFirewallUnavailablePassAsync(
|
||||||
=> link.DesiredState == "Active"
|
IReadOnlyList<LinkPolicy> candidates,
|
||||||
|| (link.DesiredState == "Disabled" && link.ActualState != "Disabled");
|
int examined,
|
||||||
|
int converged,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await MarkFirewallUnavailableAsync(candidates, cancellationToken);
|
||||||
|
events.Publish(
|
||||||
|
FirewallUnavailableCode,
|
||||||
|
"mesh",
|
||||||
|
JsonSerializer.Serialize(
|
||||||
|
new ControlError(FirewallUnavailableCode), SmmJsonContext.Default.ControlError));
|
||||||
|
var failed = examined - converged;
|
||||||
|
var failedPolicyIds = candidates
|
||||||
|
.Take(failed)
|
||||||
|
.Select(static candidate => candidate.Id)
|
||||||
|
.ToArray();
|
||||||
|
return new LinkFullReconciliationResult(
|
||||||
|
examined, converged, failed, 0, true, failedPolicyIds, []);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static LinkRule ToRule(LinkPolicy link)
|
||||||
|
=> new(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
|
|
||||||
private void Publish(string type, LinkPolicy link)
|
private void Publish(string type, LinkPolicy link)
|
||||||
=> events.Publish(
|
=> events.Publish(
|
||||||
|
|
@ -412,15 +805,50 @@ public sealed class LinkService(
|
||||||
return gate;
|
return gate;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task VerifyFactualStateAsync(
|
private async Task<IDisposable> AcquireLinkGatesAsync(
|
||||||
LinkPolicy link,
|
IEnumerable<string> ids,
|
||||||
bool expectedConnected,
|
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (await applier.IsConnectedAsync(link, cancellationToken) != expectedConnected)
|
var gates = ids
|
||||||
|
.Distinct(StringComparer.Ordinal)
|
||||||
|
.Order(StringComparer.Ordinal)
|
||||||
|
.Select(id => _reconciliationLocks.GetOrAdd(id, static _ => new SemaphoreSlim(1, 1)))
|
||||||
|
.ToArray();
|
||||||
|
var acquired = 0;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
foreach (var gate in gates)
|
||||||
|
{
|
||||||
|
await gate.WaitAsync(cancellationToken);
|
||||||
|
acquired++;
|
||||||
|
}
|
||||||
|
return new LockLease(gates);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
for (var index = acquired - 1; index >= 0; index--)
|
||||||
|
{
|
||||||
|
gates[index].Release();
|
||||||
|
}
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<int> CountExactRulesAsync(
|
||||||
|
LinkPolicy link,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
=> (await applier.ListRulesAsync(cancellationToken)).Count(rule => rule == ToRule(link));
|
||||||
|
|
||||||
|
private async Task VerifyExactFactualCountAsync(
|
||||||
|
LinkPolicy link,
|
||||||
|
int expectedCount,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var actualCount = await CountExactRulesAsync(link, cancellationToken);
|
||||||
|
if (actualCount != expectedCount)
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException(
|
throw new InvalidOperationException(
|
||||||
$"Factual Link policy is {(expectedConnected ? "disabled" : "active")} after application.");
|
$"Factual Link policy count must be exactly {expectedCount} after application, but was {actualCount}.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -438,7 +866,64 @@ public sealed class LinkService(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class FullReconciliationBatch
|
||||||
|
{
|
||||||
|
private readonly Dictionary<string, PendingConvergence> _pending =
|
||||||
|
new(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
public bool MutationAttempted { get; private set; }
|
||||||
|
public IReadOnlyCollection<PendingConvergence> Pending => _pending.Values;
|
||||||
|
|
||||||
|
public void MarkMutationAttempted() => MutationAttempted = true;
|
||||||
|
|
||||||
|
public void Stage(LinkPolicy link, bool expectedConnected, string actor, bool persisted)
|
||||||
|
=> _pending[link.Id] = new PendingConvergence(link, expectedConnected, actor, persisted);
|
||||||
|
|
||||||
|
public bool Contains(string id) => _pending.ContainsKey(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record PendingConvergence(
|
||||||
|
LinkPolicy Link,
|
||||||
|
bool ExpectedConnected,
|
||||||
|
string Actor,
|
||||||
|
bool Persisted);
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record LinkExpirationResult(int Disabled, int Failed);
|
public sealed record LinkExpirationResult(int Disabled, int Failed);
|
||||||
public sealed record LinkFullReconciliationResult(int Examined, int Failed, bool FirewallUnavailable);
|
public sealed record LinkFullReconciliationResult
|
||||||
|
{
|
||||||
|
public LinkFullReconciliationResult(
|
||||||
|
int examined,
|
||||||
|
int converged,
|
||||||
|
int failed,
|
||||||
|
int deferred,
|
||||||
|
bool firewallUnavailable,
|
||||||
|
IReadOnlyList<string> failedPolicyIds,
|
||||||
|
IReadOnlyList<string> deferredPolicyIds)
|
||||||
|
{
|
||||||
|
if (converged + failed + deferred != examined)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Full Link reconciliation classification invariant failed: examined={examined}, "
|
||||||
|
+ $"converged={converged}, failed={failed}, deferred={deferred}; "
|
||||||
|
+ $"failed IDs=[{string.Join(',', failedPolicyIds)}], "
|
||||||
|
+ $"deferred IDs=[{string.Join(',', deferredPolicyIds)}].");
|
||||||
|
}
|
||||||
|
Examined = examined;
|
||||||
|
Converged = converged;
|
||||||
|
Failed = failed;
|
||||||
|
Deferred = deferred;
|
||||||
|
FirewallUnavailable = firewallUnavailable;
|
||||||
|
FailedPolicyIds = failedPolicyIds;
|
||||||
|
DeferredPolicyIds = deferredPolicyIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Examined { get; }
|
||||||
|
public int Converged { get; }
|
||||||
|
public int Failed { get; }
|
||||||
|
public int Deferred { get; }
|
||||||
|
public bool FirewallUnavailable { get; }
|
||||||
|
public IReadOnlyList<string> FailedPolicyIds { get; }
|
||||||
|
public IReadOnlyList<string> DeferredPolicyIds { get; }
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,7 @@ builder.Services.AddOptions<ControlOptions>()
|
||||||
&& options.MetricRetentionHours is >= 24 and <= 8760
|
&& options.MetricRetentionHours is >= 24 and <= 8760
|
||||||
&& options.IdempotencyRetentionHours is >= 1 and <= 720
|
&& options.IdempotencyRetentionHours is >= 1 and <= 720
|
||||||
&& options.AuditRetentionDays is >= 1 and <= 3650
|
&& options.AuditRetentionDays is >= 1 and <= 3650
|
||||||
|
&& options.LinkRetentionDays is >= 1 and <= 3650
|
||||||
&& options.MaintenanceIntervalMinutes is >= 1 and <= 1440
|
&& options.MaintenanceIntervalMinutes is >= 1 and <= 1440
|
||||||
&& options.LinkExpirationPollSeconds is >= 1 and <= 300
|
&& options.LinkExpirationPollSeconds is >= 1 and <= 300
|
||||||
&& options.LinkReconciliationSeconds is >= 30 and <= 3600
|
&& options.LinkReconciliationSeconds is >= 30 and <= 3600
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<InvariantGlobalization>true</InvariantGlobalization>
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
|
||||||
<Version>0.1.0</Version>
|
<Version>0.1.0</Version>
|
||||||
<AssemblyName>ochenstarik-smm-control</AssemblyName>
|
<AssemblyName>ochenstarik-smm-control</AssemblyName>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
"MetricRetentionHours": 168,
|
"MetricRetentionHours": 168,
|
||||||
"IdempotencyRetentionHours": 24,
|
"IdempotencyRetentionHours": 24,
|
||||||
"AuditRetentionDays": 90,
|
"AuditRetentionDays": 90,
|
||||||
|
"LinkRetentionDays": 90,
|
||||||
"MaintenanceIntervalMinutes": 15,
|
"MaintenanceIntervalMinutes": 15,
|
||||||
"LinkExpirationPollSeconds": 15,
|
"LinkExpirationPollSeconds": 15,
|
||||||
"LinkReconciliationSeconds": 300,
|
"LinkReconciliationSeconds": 300,
|
||||||
|
|
|
||||||
|
|
@ -1059,6 +1059,7 @@ public sealed partial class MainPage : Page
|
||||||
.GroupBy(link => new { link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port })
|
.GroupBy(link => new { link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port })
|
||||||
.Select(group => group.MaxBy(link => link.Version)!)
|
.Select(group => group.MaxBy(link => link.Version)!)
|
||||||
.Any(link => link.LastError == MeshLinkViewModel.FirewallUnavailableErrorCode));
|
.Any(link => link.LastError == MeshLinkViewModel.FirewallUnavailableErrorCode));
|
||||||
|
_linksPage?.RefreshFilter();
|
||||||
|
|
||||||
var activeLinks = MeshLinks.Count(link => link.ActualState == "Active");
|
var activeLinks = MeshLinks.Count(link => link.ActualState == "Active");
|
||||||
ActiveLinksValueText.Text = activeLinks.ToString(CultureInfo.InvariantCulture);
|
ActiveLinksValueText.Text = activeLinks.ToString(CultureInfo.InvariantCulture);
|
||||||
|
|
@ -1265,10 +1266,14 @@ public sealed partial class MainPage : Page
|
||||||
await RefreshMeshAsync(showSuccess: false);
|
await RefreshMeshAsync(showSuccess: false);
|
||||||
ShowInfo(
|
ShowInfo(
|
||||||
enable ? "Control Link создан" : "Control Link отключён",
|
enable ? "Control Link создан" : "Control Link отключён",
|
||||||
$"{source.Name} → {target.Name} · {protocol.ToUpperInvariant()}/{port} · {link.ActualState} v{link.Version}",
|
link.LastError == MeshLinkViewModel.NodeNotActivatedErrorCode
|
||||||
link.ActualState is "Failed" or "Partial"
|
? $"{source.Name} → {target.Name} · {protocol.ToUpperInvariant()}/{port} · ожидает активации Node в Mesh"
|
||||||
? InfoBarSeverity.Warning
|
: $"{source.Name} → {target.Name} · {protocol.ToUpperInvariant()}/{port} · {link.ActualState} v{link.Version}",
|
||||||
: InfoBarSeverity.Success);
|
link.LastError == MeshLinkViewModel.NodeNotActivatedErrorCode
|
||||||
|
? InfoBarSeverity.Informational
|
||||||
|
: link.ActualState is "Failed" or "Partial"
|
||||||
|
? InfoBarSeverity.Warning
|
||||||
|
: InfoBarSeverity.Success);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ public sealed class MeshNodeViewModel
|
||||||
public sealed class MeshLinkViewModel
|
public sealed class MeshLinkViewModel
|
||||||
{
|
{
|
||||||
public const string FirewallUnavailableErrorCode = "mesh.firewall-unavailable";
|
public const string FirewallUnavailableErrorCode = "mesh.firewall-unavailable";
|
||||||
|
public const string NodeNotActivatedErrorCode = "mesh.node-not-activated";
|
||||||
|
|
||||||
public MeshLinkViewModel(
|
public MeshLinkViewModel(
|
||||||
string source,
|
string source,
|
||||||
|
|
@ -63,9 +64,13 @@ public sealed class MeshLinkViewModel
|
||||||
public string? LastError { get; set; }
|
public string? LastError { get; set; }
|
||||||
public bool HasDrift => !string.Equals(DesiredState, ActualState, StringComparison.Ordinal);
|
public bool HasDrift => !string.Equals(DesiredState, ActualState, StringComparison.Ordinal);
|
||||||
public string DesiredStatusText => $"Желаемое состояние: {DesiredState}";
|
public string DesiredStatusText => $"Желаемое состояние: {DesiredState}";
|
||||||
public string ActualStatusText => $"Фактическое состояние: {ActualState}";
|
public string ActualStatusText => LastError == NodeNotActivatedErrorCode
|
||||||
public string DriftText => HasDrift ? "Расхождение: требуется сверка политики" : "Расхождение: нет";
|
? "Фактическое состояние: ожидает активации Node в Mesh"
|
||||||
public string ErrorText => LastError == FirewallUnavailableErrorCode ? string.Empty
|
: $"Фактическое состояние: {ActualState}";
|
||||||
|
public string DriftText => LastError == NodeNotActivatedErrorCode
|
||||||
|
? "Ожидание: активируйте Node в Mesh — это не ошибка политики"
|
||||||
|
: HasDrift ? "Расхождение: требуется сверка политики" : "Расхождение: нет";
|
||||||
|
public string ErrorText => LastError is FirewallUnavailableErrorCode or NodeNotActivatedErrorCode ? string.Empty
|
||||||
: string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}";
|
: string.IsNullOrWhiteSpace(LastError) ? "Ошибка: нет" : $"Ошибка: {LastError}";
|
||||||
public string VersionText => $"Версия политики: {Version}";
|
public string VersionText => $"Версия политики: {Version}";
|
||||||
public string ExpirationText => ExpiresUnix == 0
|
public string ExpirationText => ExpiresUnix == 0
|
||||||
|
|
|
||||||
|
|
@ -77,8 +77,15 @@
|
||||||
|
|
||||||
<Grid x:Name="LinksListPanel" Grid.Row="1" RowSpacing="10">
|
<Grid x:Name="LinksListPanel" Grid.Row="1" RowSpacing="10">
|
||||||
<Grid.RowDefinitions><RowDefinition Height="Auto" /><RowDefinition Height="*" /></Grid.RowDefinitions>
|
<Grid.RowDefinitions><RowDefinition Height="Auto" /><RowDefinition Height="*" /></Grid.RowDefinitions>
|
||||||
<TextBlock Style="{StaticResource SubtitleTextBlockStyle}" Text="Политики направлений" />
|
<Grid ColumnSpacing="12">
|
||||||
<ListView x:Name="LinksList" Grid.Row="1" AutomationProperties.Name="Список связей" ItemsSource="{x:Bind Links}" SelectionMode="Single">
|
<Grid.ColumnDefinitions><ColumnDefinition Width="*" /><ColumnDefinition Width="Auto" /></Grid.ColumnDefinitions>
|
||||||
|
<StackPanel Spacing="2">
|
||||||
|
<TextBlock Style="{StaticResource SubtitleTextBlockStyle}" Text="Политики направлений" />
|
||||||
|
<TextBlock x:Name="LinksCountText" Foreground="{ThemeResource TextFillColorSecondaryBrush}" />
|
||||||
|
</StackPanel>
|
||||||
|
<ToggleSwitch x:Name="ShowHistoryToggle" Grid.Column="1" Header="Показать историю" Toggled="ShowHistoryToggle_Toggled" />
|
||||||
|
</Grid>
|
||||||
|
<ListView x:Name="LinksList" Grid.Row="1" AutomationProperties.Name="Список связей" ItemsSource="{x:Bind DisplayedLinks}" SelectionMode="Single">
|
||||||
<ListView.ItemTemplate>
|
<ListView.ItemTemplate>
|
||||||
<DataTemplate x:DataType="local:MeshLinkViewModel">
|
<DataTemplate x:DataType="local:MeshLinkViewModel">
|
||||||
<Grid MinHeight="62" ColumnSpacing="12">
|
<Grid MinHeight="62" ColumnSpacing="12">
|
||||||
|
|
|
||||||
|
|
@ -12,11 +12,34 @@ public sealed partial class LinksPage : Page
|
||||||
{
|
{
|
||||||
_host = host;
|
_host = host;
|
||||||
InitializeComponent();
|
InitializeComponent();
|
||||||
|
RefreshFilter();
|
||||||
}
|
}
|
||||||
|
|
||||||
public ObservableCollection<MeshNodeViewModel> Nodes => _host.MeshNodes;
|
public ObservableCollection<MeshNodeViewModel> Nodes => _host.MeshNodes;
|
||||||
|
|
||||||
public ObservableCollection<MeshLinkViewModel> Links => _host.MeshLinks;
|
public ObservableCollection<MeshLinkViewModel> DisplayedLinks { get; } = [];
|
||||||
|
|
||||||
|
internal void RefreshFilter()
|
||||||
|
{
|
||||||
|
var selectedId = (LinksList.SelectedItem as MeshLinkViewModel)?.Id;
|
||||||
|
var effective = _host.MeshLinks
|
||||||
|
.GroupBy(link => new { link.Source, link.Target, link.Protocol, link.Port })
|
||||||
|
.Select(group => group.MaxBy(link => link.Version)!)
|
||||||
|
.ToArray();
|
||||||
|
var source = ShowHistoryToggle.IsOn
|
||||||
|
? _host.MeshLinks
|
||||||
|
: effective.Where(link => link.DesiredState == "Active" || link.HasDrift);
|
||||||
|
DisplayedLinks.Clear();
|
||||||
|
foreach (var link in source)
|
||||||
|
{
|
||||||
|
DisplayedLinks.Add(link);
|
||||||
|
}
|
||||||
|
LinksCountText.Text = $"Показано политик: {DisplayedLinks.Count} · фактически Active: {DisplayedLinks.Count(link => link.ActualState == "Active")} · с расхождением: {DisplayedLinks.Count(link => link.HasDrift)}";
|
||||||
|
if (selectedId is not null)
|
||||||
|
{
|
||||||
|
LinksList.SelectedItem = DisplayedLinks.FirstOrDefault(link => link.Id == selectedId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
internal void SetFirewallUnavailable(bool unavailable)
|
internal void SetFirewallUnavailable(bool unavailable)
|
||||||
=> FirewallUnavailableInfo.IsOpen = unavailable;
|
=> FirewallUnavailableInfo.IsOpen = unavailable;
|
||||||
|
|
@ -24,6 +47,9 @@ public sealed partial class LinksPage : Page
|
||||||
private async void RefreshButton_Click(object sender, RoutedEventArgs e)
|
private async void RefreshButton_Click(object sender, RoutedEventArgs e)
|
||||||
=> await _host.RefreshLinksFromPageAsync();
|
=> await _host.RefreshLinksFromPageAsync();
|
||||||
|
|
||||||
|
private void ShowHistoryToggle_Toggled(object sender, RoutedEventArgs e)
|
||||||
|
=> RefreshFilter();
|
||||||
|
|
||||||
private async void ConnectButton_Click(object sender, RoutedEventArgs e)
|
private async void ConnectButton_Click(object sender, RoutedEventArgs e)
|
||||||
=> await ChangeLinkAsync(enable: true);
|
=> await ChangeLinkAsync(enable: true);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -109,6 +109,62 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
|
||||||
Assert.Equal(8L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
|
Assert.Equal(8L, (long)(await version.ExecuteScalarAsync(cancellationToken))!);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task MaintenanceRetainsActiveAndDriftButPrunesCompletedHistoryWithoutResurrection()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var (store, options) = CreateServices();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "E111", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "completed", "E222", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "active", "E333", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "drift", "E444", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var completed = await service.CreateAsync(
|
||||||
|
new LinkPolicyCreateRequest("source", "completed", "tcp", 22, 0, "done", Guid.NewGuid().ToString()),
|
||||||
|
"operator", cancellationToken);
|
||||||
|
await service.DisableAsync(
|
||||||
|
completed.Id, new LinkPolicyDisableRequest(Guid.NewGuid().ToString()), "operator", cancellationToken);
|
||||||
|
var active = await service.CreateAsync(
|
||||||
|
new LinkPolicyCreateRequest("source", "active", "tcp", 22, 0, "active", Guid.NewGuid().ToString()),
|
||||||
|
"operator", cancellationToken);
|
||||||
|
var drift = await service.CreateAsync(
|
||||||
|
new LinkPolicyCreateRequest("source", "drift", "tcp", 22, 0, "drift", Guid.NewGuid().ToString()),
|
||||||
|
"operator", cancellationToken);
|
||||||
|
await store.SetLinkActualStateAsync(drift.Id, "Failed", "simulated", "test", cancellationToken);
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
|
await using (var connection = new SqliteConnection($"Data Source={options.DatabasePath}"))
|
||||||
|
{
|
||||||
|
await connection.OpenAsync(cancellationToken);
|
||||||
|
var command = connection.CreateCommand();
|
||||||
|
command.CommandText = """
|
||||||
|
INSERT INTO links(
|
||||||
|
id, source_node_id, target_node_id, protocol, port, ttl_minutes, reason,
|
||||||
|
desired_state, actual_state, version, created_at, expires_at, updated_at, last_error)
|
||||||
|
SELECT $history_id, source_node_id, target_node_id, protocol, port, ttl_minutes, reason,
|
||||||
|
'Active', 'Active', version - 1, $old, NULL, $old, NULL
|
||||||
|
FROM links WHERE id = $completed_id;
|
||||||
|
UPDATE links SET updated_at = $old;
|
||||||
|
""";
|
||||||
|
command.Parameters.AddWithValue("$history_id", Guid.NewGuid().ToString());
|
||||||
|
command.Parameters.AddWithValue("$completed_id", completed.Id);
|
||||||
|
command.Parameters.AddWithValue("$old", now.AddDays(-10).ToString("O"));
|
||||||
|
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = await store.MaintainAsync(now, cancellationToken);
|
||||||
|
var remaining = await store.ListLinksAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal(2, result.LinksDeleted);
|
||||||
|
Assert.Contains(remaining, link => link.Id == active.Id && link.ActualState == "Active");
|
||||||
|
Assert.Contains(remaining, link => link.Id == drift.Id && link.ActualState == "Failed");
|
||||||
|
Assert.DoesNotContain(remaining, link => link.TargetNodeId == "completed");
|
||||||
|
Assert.DoesNotContain(await store.ListEffectiveLinksAsync(cancellationToken),
|
||||||
|
link => link.TargetNodeId == "completed");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ExpiredProvisioningJobsAreCancelledOrRequireReconciliationAndCanRetry()
|
public async Task ExpiredProvisioningJobsAreCancelledOrRequireReconciliationAndCanRetry()
|
||||||
{
|
{
|
||||||
|
|
@ -201,7 +257,8 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
|
||||||
BackupDirectory = Path.Combine(_directory, "backups"),
|
BackupDirectory = Path.Combine(_directory, "backups"),
|
||||||
MetricRetentionHours = 24,
|
MetricRetentionHours = 24,
|
||||||
IdempotencyRetentionHours = 1,
|
IdempotencyRetentionHours = 1,
|
||||||
AuditRetentionDays = 1
|
AuditRetentionDays = 1,
|
||||||
|
LinkRetentionDays = 1
|
||||||
};
|
};
|
||||||
return (new ControlStore(Options.Create(options)), options);
|
return (new ControlStore(Options.Create(options)), options);
|
||||||
}
|
}
|
||||||
|
|
@ -224,12 +281,19 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
|
||||||
|
|
||||||
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
|
private LinkRule? _connectedRule;
|
||||||
public bool FailDisconnect { get; set; }
|
public bool FailDisconnect { get; set; }
|
||||||
public int DisconnectCalls { get; private set; }
|
public int DisconnectCalls { get; private set; }
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult<IReadOnlyList<LinkRule>>(IsConnected && _connectedRule is not null
|
||||||
|
? [_connectedRule]
|
||||||
|
: []);
|
||||||
|
|
||||||
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
IsConnected = true;
|
IsConnected = true;
|
||||||
|
_connectedRule = new LinkRule(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -244,6 +308,13 @@ public sealed class ControlMaintenanceTests : IAsyncDisposable
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
DisconnectCalls++;
|
||||||
|
IsConnected = false;
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
private bool IsConnected { get; set; }
|
private bool IsConnected { get; set; }
|
||||||
|
|
||||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
|
|
|
||||||
|
|
@ -462,7 +462,8 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
|
|
||||||
applier.IsConnected = false;
|
applier.IsConnected = false;
|
||||||
var activeResult = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
var activeResult = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 0), activeResult);
|
Assert.Equal((1, 1, 0, 0),
|
||||||
|
(activeResult.Examined, activeResult.Converged, activeResult.Failed, activeResult.Deferred));
|
||||||
Assert.Equal(3, applier.ConnectCalls);
|
Assert.Equal(3, applier.ConnectCalls);
|
||||||
|
|
||||||
var latest = (await store.ListEffectiveLinksForNodeAsync("home", cancellationToken)).Single();
|
var latest = (await store.ListEffectiveLinksForNodeAsync("home", cancellationToken)).Single();
|
||||||
|
|
@ -475,7 +476,8 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
|
|
||||||
var disabledResult = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
var disabledResult = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
||||||
|
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 0), disabledResult);
|
Assert.Equal((1, 1, 0, 0),
|
||||||
|
(disabledResult.Examined, disabledResult.Converged, disabledResult.Failed, disabledResult.Deferred));
|
||||||
Assert.Equal(beforeReconnect, applier.DisconnectCalls);
|
Assert.Equal(beforeReconnect, applier.DisconnectCalls);
|
||||||
var persisted = (await store.ListEffectiveLinksForNodeAsync("home", cancellationToken)).Single();
|
var persisted = (await store.ListEffectiveLinksForNodeAsync("home", cancellationToken)).Single();
|
||||||
Assert.Equal("Disabled", persisted.DesiredState);
|
Assert.Equal("Disabled", persisted.DesiredState);
|
||||||
|
|
@ -512,7 +514,8 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
applier.FailDisconnect = true;
|
applier.FailDisconnect = true;
|
||||||
applier.IsConnected = true;
|
applier.IsConnected = true;
|
||||||
var failed = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
var failed = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 1), failed);
|
Assert.Equal((1, 0, 1, 0),
|
||||||
|
(failed.Examined, failed.Converged, failed.Failed, failed.Deferred));
|
||||||
var retry = await store.RecordHeartbeatAsync(
|
var retry = await store.RecordHeartbeatAsync(
|
||||||
heartbeat with
|
heartbeat with
|
||||||
{
|
{
|
||||||
|
|
@ -525,7 +528,8 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
|
|
||||||
applier.FailDisconnect = false;
|
applier.FailDisconnect = false;
|
||||||
var succeeded = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
var succeeded = await service.ReconcileLinksForNodeAsync("home", cancellationToken);
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 0), succeeded);
|
Assert.Equal((1, 1, 0, 0),
|
||||||
|
(succeeded.Examined, succeeded.Converged, succeeded.Failed, succeeded.Deferred));
|
||||||
await store.CompleteAgentReconciliationAsync("home", cancellationToken);
|
await store.CompleteAgentReconciliationAsync("home", cancellationToken);
|
||||||
var completed = await store.RecordHeartbeatAsync(
|
var completed = await store.RecordHeartbeatAsync(
|
||||||
heartbeat with
|
heartbeat with
|
||||||
|
|
@ -964,12 +968,17 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
|
|
||||||
private sealed class CheckingPolicyApplier(ControlStore store) : ILinkPolicyApplier
|
private sealed class CheckingPolicyApplier(ControlStore store) : ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
|
private LinkRule? _connectedRule;
|
||||||
public int ConnectCalls { get; private set; }
|
public int ConnectCalls { get; private set; }
|
||||||
public int DisconnectCalls { get; private set; }
|
public int DisconnectCalls { get; private set; }
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult<IReadOnlyList<LinkRule>>(_connectedRule is null ? [] : [_connectedRule]);
|
||||||
|
|
||||||
public async Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public async Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
ConnectCalls++;
|
ConnectCalls++;
|
||||||
|
_connectedRule = new LinkRule(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
||||||
Assert.Equal(link.Id, persisted.Id);
|
Assert.Equal(link.Id, persisted.Id);
|
||||||
Assert.Equal("Active", persisted.DesiredState);
|
Assert.Equal("Active", persisted.DesiredState);
|
||||||
|
|
@ -979,28 +988,45 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
public async Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public async Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
DisconnectCalls++;
|
DisconnectCalls++;
|
||||||
|
_connectedRule = null;
|
||||||
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
var persisted = Assert.Single(await store.ListLinksAsync(cancellationToken));
|
||||||
Assert.Equal(link.Id, persisted.Id);
|
Assert.Equal(link.Id, persisted.Id);
|
||||||
Assert.Equal("Disabled", persisted.DesiredState);
|
Assert.Equal("Disabled", persisted.DesiredState);
|
||||||
Assert.Equal("Disconnecting", persisted.ActualState);
|
Assert.Equal("Disconnecting", persisted.ActualState);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
DisconnectCalls++;
|
||||||
|
_connectedRule = null;
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
=> Task.FromResult(ConnectCalls > DisconnectCalls);
|
=> Task.FromResult(_connectedRule is not null);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class CountingPolicyApplier : ILinkPolicyApplier
|
private sealed class CountingPolicyApplier : ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
|
private LinkRule? _lastRule;
|
||||||
public int ConnectCalls { get; private set; }
|
public int ConnectCalls { get; private set; }
|
||||||
public int DisconnectCalls { get; private set; }
|
public int DisconnectCalls { get; private set; }
|
||||||
public bool FailDisconnect { get; set; }
|
public bool FailDisconnect { get; set; }
|
||||||
public bool ConnectLeavesRuleAbsent { get; set; }
|
public bool ConnectLeavesRuleAbsent { get; set; }
|
||||||
public bool IsConnected { get; set; }
|
public bool IsConnected { get; set; }
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult<IReadOnlyList<LinkRule>>(IsConnected && _lastRule is not null
|
||||||
|
? [_lastRule]
|
||||||
|
: []);
|
||||||
|
|
||||||
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
ConnectCalls++;
|
ConnectCalls++;
|
||||||
IsConnected = !ConnectLeavesRuleAbsent;
|
IsConnected = !ConnectLeavesRuleAbsent;
|
||||||
|
_lastRule = IsConnected
|
||||||
|
? new LinkRule(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port)
|
||||||
|
: null;
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1015,6 +1041,13 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
DisconnectCalls++;
|
||||||
|
IsConnected = false;
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
=> Task.FromResult(IsConnected);
|
=> Task.FromResult(IsConnected);
|
||||||
}
|
}
|
||||||
|
|
@ -1023,10 +1056,16 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
{
|
{
|
||||||
private TaskCompletionSource _connectStarted = CompletedSource();
|
private TaskCompletionSource _connectStarted = CompletedSource();
|
||||||
private TaskCompletionSource _releaseConnect = CompletedSource();
|
private TaskCompletionSource _releaseConnect = CompletedSource();
|
||||||
|
private LinkRule? _lastRule;
|
||||||
|
|
||||||
public bool IsConnected { get; set; }
|
public bool IsConnected { get; set; }
|
||||||
public Task ConnectStarted => _connectStarted.Task;
|
public Task ConnectStarted => _connectStarted.Task;
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult<IReadOnlyList<LinkRule>>(IsConnected && _lastRule is not null
|
||||||
|
? [_lastRule]
|
||||||
|
: []);
|
||||||
|
|
||||||
public void BlockNextConnect()
|
public void BlockNextConnect()
|
||||||
{
|
{
|
||||||
_connectStarted = NewSource();
|
_connectStarted = NewSource();
|
||||||
|
|
@ -1040,6 +1079,7 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
_connectStarted.TrySetResult();
|
_connectStarted.TrySetResult();
|
||||||
await _releaseConnect.Task.WaitAsync(cancellationToken);
|
await _releaseConnect.Task.WaitAsync(cancellationToken);
|
||||||
IsConnected = true;
|
IsConnected = true;
|
||||||
|
_lastRule = new LinkRule(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
|
|
@ -1048,6 +1088,12 @@ public sealed class ControlStoreTests : IAsyncDisposable
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
IsConnected = false;
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
=> Task.FromResult(IsConnected);
|
=> Task.FromResult(IsConnected);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
var failureMarkerPath = Path.Combine(_directory, "fail-disconnect");
|
var failureMarkerPath = Path.Combine(_directory, "fail-disconnect");
|
||||||
var firewallUnavailableMarkerPath = Path.Combine(_directory, "firewall-unavailable");
|
var firewallUnavailableMarkerPath = Path.Combine(_directory, "firewall-unavailable");
|
||||||
var connectedMarkerPath = Path.Combine(_directory, "connected");
|
var connectedMarkerPath = Path.Combine(_directory, "connected");
|
||||||
|
var orphanMarkerPath = Path.Combine(_directory, "orphan-present");
|
||||||
var invocationLogPath = Path.Combine(_directory, "helper.log");
|
var invocationLogPath = Path.Combine(_directory, "helper.log");
|
||||||
await WriteExecutableAsync(
|
await WriteExecutableAsync(
|
||||||
sudoPath,
|
sudoPath,
|
||||||
|
|
@ -56,7 +57,23 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
exit 23
|
exit 23
|
||||||
fi
|
fi
|
||||||
if [ "${1:-}" = "link-disconnect" ]; then
|
if [ "${1:-}" = "link-disconnect" ]; then
|
||||||
rm -f '{{ShellQuote(connectedMarkerPath)}}'
|
if [ "${2:-}" = "orphan" ]; then
|
||||||
|
rm -f '{{ShellQuote(orphanMarkerPath)}}'
|
||||||
|
else
|
||||||
|
rm -f '{{ShellQuote(connectedMarkerPath)}}'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ "${1:-}" = "link-list" ]; then
|
||||||
|
if [ -f '{{ShellQuote(firewallUnavailableMarkerPath)}}' ]; then
|
||||||
|
echo "mesh.firewall-unavailable" >&2
|
||||||
|
exit 79
|
||||||
|
fi
|
||||||
|
if [ -f '{{ShellQuote(connectedMarkerPath)}}' ]; then
|
||||||
|
printf 'ai-agent\thome\ttcp\t22\n'
|
||||||
|
fi
|
||||||
|
if [ -f '{{ShellQuote(orphanMarkerPath)}}' ]; then
|
||||||
|
printf 'orphan\tmissing\ttcp\t23\n'
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
if [ "${1:-}" = "link-status" ]; then
|
if [ "${1:-}" = "link-status" ]; then
|
||||||
if [ -f '{{ShellQuote(firewallUnavailableMarkerPath)}}' ]; then
|
if [ -f '{{ShellQuote(firewallUnavailableMarkerPath)}}' ]; then
|
||||||
|
|
@ -91,19 +108,43 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
Assert.Equal("Active", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
Assert.Equal("Active", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
||||||
Assert.Equal(
|
Assert.Equal(
|
||||||
[
|
[
|
||||||
"link-status ai-agent home tcp 22",
|
"link-list",
|
||||||
"link-connect ai-agent home tcp 22 60",
|
"link-connect ai-agent home tcp 22 60",
|
||||||
"link-status ai-agent home tcp 22"
|
"link-list"
|
||||||
],
|
],
|
||||||
await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
||||||
|
|
||||||
|
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||||
|
var noDrift = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
Assert.Equal((1, 1, 0), (noDrift.Examined, noDrift.Converged, noDrift.Failed));
|
||||||
|
Assert.Equal(["link-list"], await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
||||||
|
|
||||||
|
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||||
await File.WriteAllTextAsync(firewallUnavailableMarkerPath, "fail", cancellationToken);
|
await File.WriteAllTextAsync(firewallUnavailableMarkerPath, "fail", cancellationToken);
|
||||||
var unavailable = await service.ReconcileAllAsync(cancellationToken);
|
var unavailable = await service.ReconcileAllAsync(cancellationToken);
|
||||||
Assert.True(unavailable.FirewallUnavailable);
|
Assert.True(unavailable.FirewallUnavailable);
|
||||||
|
Assert.Equal(["link-list"], await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
||||||
Assert.Equal(LinkService.FirewallUnavailableCode,
|
Assert.Equal(LinkService.FirewallUnavailableCode,
|
||||||
(await store.GetLinkAsync(active.Id, cancellationToken))!.LastError);
|
(await store.GetLinkAsync(active.Id, cancellationToken))!.LastError);
|
||||||
File.Delete(firewallUnavailableMarkerPath);
|
File.Delete(firewallUnavailableMarkerPath);
|
||||||
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
|
Assert.False((await service.ReconcileAllAsync(cancellationToken)).FirewallUnavailable);
|
||||||
|
File.Delete(connectedMarkerPath);
|
||||||
|
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||||
|
|
||||||
|
var orphanStore = new ControlStore(Options.Create(new ControlOptions
|
||||||
|
{
|
||||||
|
DatabasePath = Path.Combine(_directory, "orphan-control.db"),
|
||||||
|
CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx")
|
||||||
|
}));
|
||||||
|
await orphanStore.InitializeAsync(cancellationToken);
|
||||||
|
var orphanService = CreateLinkService(orphanStore, sudoPath, helperPath);
|
||||||
|
await File.WriteAllTextAsync(orphanMarkerPath, "present", cancellationToken);
|
||||||
|
var orphanResult = await orphanService.ReconcileAllAsync(cancellationToken);
|
||||||
|
Assert.Equal((1, 1, 0), (orphanResult.Examined, orphanResult.Converged, orphanResult.Failed));
|
||||||
|
Assert.Equal(
|
||||||
|
["link-list", "link-disconnect orphan missing tcp 23", "link-list"],
|
||||||
|
await File.ReadAllLinesAsync(invocationLogPath, cancellationToken));
|
||||||
|
await File.WriteAllTextAsync(connectedMarkerPath, "connected", cancellationToken);
|
||||||
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
await File.WriteAllTextAsync(invocationLogPath, string.Empty, cancellationToken);
|
||||||
|
|
||||||
await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken);
|
await File.WriteAllTextAsync(failureMarkerPath, "fail", cancellationToken);
|
||||||
|
|
@ -122,7 +163,9 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
var restartedService = CreateLinkService(restartedStore, sudoPath, helperPath);
|
var restartedService = CreateLinkService(restartedStore, sudoPath, helperPath);
|
||||||
var failedReconciliation = await restartedService.ReconcileLinksForNodeAsync(
|
var failedReconciliation = await restartedService.ReconcileLinksForNodeAsync(
|
||||||
"home", cancellationToken);
|
"home", cancellationToken);
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 1), failedReconciliation);
|
Assert.Equal((1, 0, 1, 0),
|
||||||
|
(failedReconciliation.Examined, failedReconciliation.Converged,
|
||||||
|
failedReconciliation.Failed, failedReconciliation.Deferred));
|
||||||
|
|
||||||
File.Delete(failureMarkerPath);
|
File.Delete(failureMarkerPath);
|
||||||
var secondRestartStore = CreateStore();
|
var secondRestartStore = CreateStore();
|
||||||
|
|
@ -130,7 +173,9 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
var secondRestartService = CreateLinkService(secondRestartStore, sudoPath, helperPath);
|
var secondRestartService = CreateLinkService(secondRestartStore, sudoPath, helperPath);
|
||||||
var successfulReconciliation = await secondRestartService.ReconcileLinksForNodeAsync(
|
var successfulReconciliation = await secondRestartService.ReconcileLinksForNodeAsync(
|
||||||
"home", cancellationToken);
|
"home", cancellationToken);
|
||||||
Assert.Equal(new LinkReconciliationResult(1, 0), successfulReconciliation);
|
Assert.Equal((1, 1, 0, 0),
|
||||||
|
(successfulReconciliation.Examined, successfulReconciliation.Converged,
|
||||||
|
successfulReconciliation.Failed, successfulReconciliation.Deferred));
|
||||||
var persisted = Assert.Single(await secondRestartStore.ListEffectiveLinksForNodeAsync(
|
var persisted = Assert.Single(await secondRestartStore.ListEffectiveLinksForNodeAsync(
|
||||||
"home", cancellationToken));
|
"home", cancellationToken));
|
||||||
Assert.Equal("Disabled", persisted.DesiredState);
|
Assert.Equal("Disabled", persisted.DesiredState);
|
||||||
|
|
@ -138,13 +183,13 @@ public sealed class LinkPolicyApplierIntegrationTests : IAsyncDisposable
|
||||||
|
|
||||||
var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken);
|
var invocations = await File.ReadAllLinesAsync(invocationLogPath, cancellationToken);
|
||||||
Assert.Equal(7, invocations.Length);
|
Assert.Equal(7, invocations.Length);
|
||||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[0]);
|
Assert.Equal("link-list", invocations[0]);
|
||||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[1]);
|
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[1]);
|
||||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[2]);
|
Assert.Equal("link-list", invocations[2]);
|
||||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[3]);
|
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[3]);
|
||||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[4]);
|
Assert.Equal("link-list", invocations[4]);
|
||||||
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[5]);
|
Assert.Equal("link-disconnect ai-agent home tcp 22", invocations[5]);
|
||||||
Assert.Equal("link-status ai-agent home tcp 22", invocations[6]);
|
Assert.Equal("link-list", invocations[6]);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ValueTask DisposeAsync()
|
public ValueTask DisposeAsync()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
using Microsoft.Data.Sqlite;
|
using Microsoft.Data.Sqlite;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.Extensions.Logging.Abstractions;
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
|
using System.Text.Json;
|
||||||
using ServerMonitorManager.Control;
|
using ServerMonitorManager.Control;
|
||||||
using ServerMonitorManager.Core;
|
using ServerMonitorManager.Core;
|
||||||
using Xunit;
|
using Xunit;
|
||||||
|
|
@ -12,6 +14,20 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
private readonly string _directory = Path.Combine(
|
private readonly string _directory = Path.Combine(
|
||||||
Path.GetTempPath(), $"smm-link-reconciliation-{Guid.NewGuid():N}");
|
Path.GetTempPath(), $"smm-link-reconciliation-{Guid.NewGuid():N}");
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ReconciliationResultsRejectNonExhaustiveClassificationsWithIds()
|
||||||
|
{
|
||||||
|
var full = Assert.Throws<InvalidOperationException>(() =>
|
||||||
|
new LinkFullReconciliationResult(6, 2, 2, 1, false, ["failed"], ["deferred"]));
|
||||||
|
Assert.Contains("examined=6", full.Message, StringComparison.Ordinal);
|
||||||
|
Assert.Contains("failed IDs=[failed]", full.Message, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
var node = Assert.Throws<InvalidOperationException>(() =>
|
||||||
|
new LinkReconciliationResult(3, 1, 0, 1, [], ["deferred"]));
|
||||||
|
Assert.Contains("examined=3", node.Message, StringComparison.Ordinal);
|
||||||
|
Assert.Contains("deferred IDs=[deferred]", node.Message, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task AllPolicyPassReappliesErasedActiveRulesWithoutHeartbeat()
|
public async Task AllPolicyPassReappliesErasedActiveRulesWithoutHeartbeat()
|
||||||
{
|
{
|
||||||
|
|
@ -54,10 +70,323 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
await service.ReconcileAllAsync(cancellationToken);
|
await service.ReconcileAllAsync(cancellationToken);
|
||||||
var connects = applier.ConnectCalls;
|
var connects = applier.ConnectCalls;
|
||||||
var disconnects = applier.DisconnectCalls;
|
var disconnects = applier.DisconnectCalls;
|
||||||
|
var privilegedBefore = applier.PrivilegedCalls;
|
||||||
await service.ReconcileAllAsync(cancellationToken);
|
await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
Assert.Equal(connects, applier.ConnectCalls);
|
Assert.Equal(connects, applier.ConnectCalls);
|
||||||
Assert.Equal(disconnects, applier.DisconnectCalls);
|
Assert.Equal(disconnects, applier.DisconnectCalls);
|
||||||
|
Assert.Equal(1, applier.PrivilegedCalls - privilegedBefore);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FullPassWithMultipleMutationsUsesOneInitialAndOneFinalList()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "A011", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
for (var index = 1; index <= 4; index++)
|
||||||
|
{
|
||||||
|
var target = $"target-{index}";
|
||||||
|
await EnrollAgentAsync(store, target, $"B0{index}2", cancellationToken);
|
||||||
|
await service.CreateAsync(CreateRequest(target), "operator", cancellationToken);
|
||||||
|
}
|
||||||
|
applier.EraseRules();
|
||||||
|
var listsBefore = applier.ListCalls;
|
||||||
|
var connectsBefore = applier.ConnectCalls;
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal((4, 4, 0, 0),
|
||||||
|
(result.Examined, result.Converged, result.Failed, result.Deferred));
|
||||||
|
Assert.Equal(2, applier.ListCalls - listsBefore);
|
||||||
|
Assert.Equal(4, applier.ConnectCalls - connectsBefore);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BatchFinalizationDoesNotFinalizeOrPublishStalePolicyVersion()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "A021", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "B032", cancellationToken);
|
||||||
|
var broker = new ControlEventBroker();
|
||||||
|
using var subscription = broker.Subscribe();
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, broker);
|
||||||
|
var link = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
applier.EraseRules();
|
||||||
|
while (subscription.Reader.TryRead(out _)) { }
|
||||||
|
var listsBefore = applier.ListCalls;
|
||||||
|
applier.PauseOnListCall = listsBefore + 2;
|
||||||
|
|
||||||
|
var pass = service.ReconcileAllAsync(cancellationToken);
|
||||||
|
await applier.PausedListObserved.Task.WaitAsync(cancellationToken);
|
||||||
|
var disabled = await store.BeginDisableLinkMutationAsync(
|
||||||
|
link.Id,
|
||||||
|
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
|
||||||
|
"operator",
|
||||||
|
cancellationToken);
|
||||||
|
Assert.NotNull(disabled);
|
||||||
|
Assert.True(disabled.Link.Version > link.Version);
|
||||||
|
applier.ReleasePausedList.TrySetResult();
|
||||||
|
|
||||||
|
var result = await pass;
|
||||||
|
|
||||||
|
Assert.Equal((1, 0, 1, 0),
|
||||||
|
(result.Examined, result.Converged, result.Failed, result.Deferred));
|
||||||
|
Assert.Equal(2, applier.ListCalls - listsBefore);
|
||||||
|
var persisted = Assert.IsType<LinkPolicy>(await store.GetLinkAsync(link.Id, cancellationToken));
|
||||||
|
Assert.Equal("Disabled", persisted.DesiredState);
|
||||||
|
Assert.Equal("Disconnecting", persisted.ActualState);
|
||||||
|
var eventTypes = new List<string>();
|
||||||
|
while (subscription.Reader.TryRead(out var controlEvent))
|
||||||
|
{
|
||||||
|
eventTypes.Add(controlEvent.Type);
|
||||||
|
}
|
||||||
|
Assert.DoesNotContain("link.active", eventTypes);
|
||||||
|
Assert.DoesNotContain("link.reapplied", eventTypes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FactualOrphanWithoutDatabasePolicyIsRemovedAndAudited()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var broker = new ControlEventBroker();
|
||||||
|
using var subscription = broker.Subscribe();
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var orphan = new LinkRule("source", "missing", "tcp", 22);
|
||||||
|
applier.InjectRule(orphan);
|
||||||
|
var service = new LinkService(store, applier, broker);
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal((1, 1, 0), (result.Examined, result.Converged, result.Failed));
|
||||||
|
Assert.Equal(1, applier.DisconnectCalls);
|
||||||
|
Assert.DoesNotContain(orphan, await applier.ListRulesAsync(cancellationToken));
|
||||||
|
var eventTypes = new List<string>();
|
||||||
|
while (subscription.Reader.TryRead(out var controlEvent))
|
||||||
|
{
|
||||||
|
eventTypes.Add(controlEvent.Type);
|
||||||
|
}
|
||||||
|
Assert.Contains("link.orphan-removed", eventTypes);
|
||||||
|
|
||||||
|
await using var connection = new SqliteConnection($"Data Source={Path.Combine(_directory, "control.db")}");
|
||||||
|
await connection.OpenAsync(cancellationToken);
|
||||||
|
var command = connection.CreateCommand();
|
||||||
|
command.CommandText = """
|
||||||
|
SELECT details_json FROM audit
|
||||||
|
WHERE action = 'link.orphan-removed'
|
||||||
|
ORDER BY sequence DESC LIMIT 1;
|
||||||
|
""";
|
||||||
|
var payload = Assert.IsType<string>(await command.ExecuteScalarAsync(cancellationToken));
|
||||||
|
using var document = JsonDocument.Parse(payload);
|
||||||
|
Assert.Equal("source", document.RootElement.GetProperty("sourceNodeId").GetString());
|
||||||
|
Assert.Equal("missing", document.RootElement.GetProperty("targetNodeId").GetString());
|
||||||
|
Assert.Equal("tcp", document.RootElement.GetProperty("protocol").GetString());
|
||||||
|
Assert.Equal(22, document.RootElement.GetProperty("port").GetInt32());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DatabaseLessOrphanIsFailedWhenDisconnectReportsSuccessButRuleRemains()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var rule = new LinkRule("source", "missing", "tcp", 22);
|
||||||
|
var applier = new RecordingPolicyApplier { DisconnectLeavesRules = true };
|
||||||
|
applier.InjectRule(rule);
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal((1, 0, 1), (result.Examined, result.Converged, result.Failed));
|
||||||
|
Assert.Contains(rule, await applier.ListRulesAsync(cancellationToken));
|
||||||
|
Assert.Contains(result.FailedPolicyIds, id => id.StartsWith("orphan:", StringComparison.Ordinal));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task PersistedDisabledCleanupDoesNotProbeMissingNodeStatus()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "D311", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "D322", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var link = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
await service.DisableAsync(
|
||||||
|
link.Id, new LinkPolicyDisableRequest(Guid.NewGuid().ToString()), "operator", cancellationToken);
|
||||||
|
applier.InjectRule(new LinkRule("source", "target-one", "tcp", 22));
|
||||||
|
applier.ThrowNodeNotActivatedOnStatus = true;
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal((1, 1, 0), (result.Examined, result.Converged, result.Failed));
|
||||||
|
Assert.Equal("Disabled", (await store.GetLinkAsync(link.Id, cancellationToken))!.ActualState);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ExactPostMutationCountRejectsDuplicateActiveRule()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "D411", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "D422", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier { ConnectAddsDuplicate = true };
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
|
||||||
|
var link = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal("Failed", link.ActualState);
|
||||||
|
Assert.Contains("exactly 1", link.LastError, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task MarkerIsRetainedWhenPostMutationFactCannotBeVerified()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var rule = new LinkRule("source", "missing", "tcp", 22);
|
||||||
|
var applier = new RecordingPolicyApplier
|
||||||
|
{
|
||||||
|
DisconnectLeavesRules = true,
|
||||||
|
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||||||
|
};
|
||||||
|
applier.InjectRule(rule);
|
||||||
|
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var background = CreateBackgroundService(
|
||||||
|
links,
|
||||||
|
applier,
|
||||||
|
new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z")));
|
||||||
|
|
||||||
|
var result = await background.RunOnceAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.NotNull(result);
|
||||||
|
Assert.Equal((1, 0, 1), (result.Examined, result.Converged, result.Failed));
|
||||||
|
Assert.Equal(0, applier.CompleteReconciliationCalls);
|
||||||
|
Assert.NotNull(applier.ReconciliationRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task MixedPassClassifiesEveryExaminedPolicyExactlyOnce()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "C011", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var links = new List<LinkPolicy>();
|
||||||
|
for (var index = 1; index <= 6; index++)
|
||||||
|
{
|
||||||
|
var target = $"mixed-{index}";
|
||||||
|
await EnrollAgentAsync(store, target, $"C0{index}2", cancellationToken);
|
||||||
|
links.Add(await service.CreateAsync(CreateRequest(target), "operator", cancellationToken));
|
||||||
|
}
|
||||||
|
applier.EraseRules();
|
||||||
|
applier.InjectRule(ToRule(links[0]));
|
||||||
|
applier.InjectRule(ToRule(links[1]));
|
||||||
|
applier.FailedMutationRules.Add(ToRule(links[2]));
|
||||||
|
applier.FailedMutationRules.Add(ToRule(links[3]));
|
||||||
|
applier.DeferredConnectRules.Add(ToRule(links[4]));
|
||||||
|
applier.DeferredConnectRules.Add(ToRule(links[5]));
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal(6, result.Examined);
|
||||||
|
Assert.Equal((2, 2, 2), (result.Converged, result.Failed, result.Deferred));
|
||||||
|
Assert.Equal(result.Examined, result.Converged + result.Failed + result.Deferred);
|
||||||
|
Assert.Equal(2, result.FailedPolicyIds.Count);
|
||||||
|
Assert.Equal(2, result.DeferredPolicyIds.Count);
|
||||||
|
Assert.Empty(result.FailedPolicyIds.Intersect(result.DeferredPolicyIds));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DeferredPoliciesConsumeMarkerAndDoNotCreatePromptHotLoop()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "D011", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "reserved", "D022", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier
|
||||||
|
{
|
||||||
|
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||||||
|
};
|
||||||
|
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var policy = await links.CreateAsync(CreateRequest("reserved"), "operator", cancellationToken);
|
||||||
|
applier.EraseRules();
|
||||||
|
applier.DeferredConnectRules.Add(ToRule(policy));
|
||||||
|
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-04T12:00:00Z"));
|
||||||
|
var logger = new TestLogger<LinkReconciliationBackgroundService>();
|
||||||
|
var background = CreateBackgroundService(links, applier, time, logger);
|
||||||
|
|
||||||
|
for (var index = 0; index < 10; index++)
|
||||||
|
{
|
||||||
|
var result = await background.RunOnceAsync(cancellationToken);
|
||||||
|
Assert.NotNull(result);
|
||||||
|
Assert.Equal((1, 0, 0, 1),
|
||||||
|
(result.Examined, result.Converged, result.Failed, result.Deferred));
|
||||||
|
Assert.Equal([policy.Id], result.DeferredPolicyIds);
|
||||||
|
if (index == 0)
|
||||||
|
{
|
||||||
|
Assert.Null(applier.ReconciliationRequest);
|
||||||
|
Assert.Equal(1, applier.CompleteReconciliationCalls);
|
||||||
|
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
time.Advance(TimeSpan.FromSeconds(30));
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.DoesNotContain(logger.Warnings,
|
||||||
|
warning => warning.Contains("prompt exhausted", StringComparison.Ordinal));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DuplicateManagedRulesCollapseForActiveAndAreRemovedForDisabled()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "D111", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "D222", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var link = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
applier.InjectRule(new LinkRule("source", "target-one", "tcp", 22));
|
||||||
|
|
||||||
|
await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
Assert.Equal(1, applier.RuleCount(link));
|
||||||
|
|
||||||
|
await service.DisableAsync(
|
||||||
|
link.Id, new LinkPolicyDisableRequest(Guid.NewGuid().ToString()), "operator", cancellationToken);
|
||||||
|
applier.InjectRule(new LinkRule("source", "target-one", "tcp", 22), 2);
|
||||||
|
await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
Assert.Equal(0, applier.RuleCount(link));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ForeignFirewallRulesAreOutsideManagedListingAndRemainUntouched()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier { ForeignRuleCount = 1 };
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.Equal(0, result.Examined);
|
||||||
|
Assert.Equal(1, applier.ForeignRuleCount);
|
||||||
|
Assert.Equal(0, applier.DisconnectCalls);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|
@ -73,15 +402,17 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
var applier = new RecordingPolicyApplier();
|
var applier = new RecordingPolicyApplier();
|
||||||
var service = new LinkService(store, applier, broker);
|
var service = new LinkService(store, applier, broker);
|
||||||
var active = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
var active = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
await store.BeginDisableLinkMutationAsync(
|
await service.DisableAsync(
|
||||||
active.Id,
|
active.Id,
|
||||||
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
|
new LinkPolicyDisableRequest(Guid.NewGuid().ToString()),
|
||||||
"operator",
|
"operator",
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
var disconnectsBefore = applier.DisconnectCalls;
|
||||||
|
applier.InjectRule(new LinkRule("source", "target-one", "tcp", 22));
|
||||||
|
|
||||||
await service.ReconcileAllAsync(cancellationToken);
|
await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
Assert.Equal(1, applier.DisconnectCalls);
|
Assert.Equal(disconnectsBefore + 1, applier.DisconnectCalls);
|
||||||
Assert.Equal("Disabled", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
Assert.Equal("Disabled", (await store.GetLinkAsync(active.Id, cancellationToken))!.ActualState);
|
||||||
var eventTypes = new List<string>();
|
var eventTypes = new List<string>();
|
||||||
while (subscription.Reader.TryRead(out var controlEvent))
|
while (subscription.Reader.TryRead(out var controlEvent))
|
||||||
|
|
@ -138,6 +469,74 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
Assert.Single(eventTypes, eventType => eventType == LinkService.FirewallAvailableCode);
|
Assert.Single(eventTypes, eventType => eventType == LinkService.FirewallAvailableCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FirewallUnavailableDuringFinalBatchListMarksWholePassAndPublishesOneSharedEvent()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "F111", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "F222", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-two", "F333", cancellationToken);
|
||||||
|
var broker = new ControlEventBroker();
|
||||||
|
using var subscription = broker.Subscribe();
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var service = new LinkService(store, applier, broker);
|
||||||
|
var first = await service.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
var second = await service.CreateAsync(CreateRequest("target-two"), "operator", cancellationToken);
|
||||||
|
applier.EraseRules();
|
||||||
|
applier.UnavailableOnNextPostMutationList = true;
|
||||||
|
while (subscription.Reader.TryRead(out _)) { }
|
||||||
|
|
||||||
|
var result = await service.ReconcileAllAsync(cancellationToken);
|
||||||
|
|
||||||
|
Assert.True(result.FirewallUnavailable);
|
||||||
|
Assert.Equal(1, applier.MutationCalls(first));
|
||||||
|
Assert.Equal(1, applier.MutationCalls(second));
|
||||||
|
Assert.All(await store.ListEffectiveLinksAsync(cancellationToken), link =>
|
||||||
|
{
|
||||||
|
Assert.Equal("Partial", link.ActualState);
|
||||||
|
Assert.Equal(LinkService.FirewallUnavailableCode, link.LastError);
|
||||||
|
});
|
||||||
|
var eventTypes = new List<string>();
|
||||||
|
while (subscription.Reader.TryRead(out var controlEvent))
|
||||||
|
{
|
||||||
|
eventTypes.Add(controlEvent.Type);
|
||||||
|
}
|
||||||
|
Assert.Single(eventTypes, eventType => eventType == LinkService.FirewallUnavailableCode);
|
||||||
|
Assert.Equal(2, eventTypes.Count(eventType => eventType == "link.reconciling"));
|
||||||
|
Assert.DoesNotContain("link.reapplied", eventTypes);
|
||||||
|
Assert.DoesNotContain("link.orphan-removed", eventTypes);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FactualOrphanAdoptedAsActiveUnderNodeLocksIsNotRemoved()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "F444", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "F555", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier();
|
||||||
|
var rule = new LinkRule("source", "target-one", "tcp", 22);
|
||||||
|
applier.InjectRule(rule);
|
||||||
|
var service = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var nodeLease = await service.AcquireNodeLocksAsync(
|
||||||
|
[rule.SourceNodeId, rule.TargetNodeId], cancellationToken);
|
||||||
|
var pass = service.ReconcileAllAsync(cancellationToken);
|
||||||
|
await applier.ListObserved.Task.WaitAsync(cancellationToken);
|
||||||
|
var adopted = (await store.CreateLinkMutationAsync(
|
||||||
|
CreateRequest("target-one"), "operator", cancellationToken)).Link;
|
||||||
|
nodeLease.Dispose();
|
||||||
|
|
||||||
|
var result = await pass;
|
||||||
|
|
||||||
|
Assert.Equal(0, result.Failed);
|
||||||
|
Assert.Equal("Active", (await store.GetLinkAsync(adopted.Id, cancellationToken))!.ActualState);
|
||||||
|
Assert.Equal(0, applier.DisconnectCalls);
|
||||||
|
Assert.Equal(1, applier.RuleCount(adopted));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task MarkerCreatedAfterNormalPassTriggersPromptAdditionalPass()
|
public async Task MarkerCreatedAfterNormalPassTriggersPromptAdditionalPass()
|
||||||
{
|
{
|
||||||
|
|
@ -193,6 +592,88 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
Assert.Null(applier.ReconciliationRequest);
|
Assert.Null(applier.ReconciliationRequest);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FailingMarkerPassIsPromptedThreeTimesThenUsesRegularThrottle()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "source", "A477", cancellationToken);
|
||||||
|
await EnrollAgentAsync(store, "target-one", "B588", cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier
|
||||||
|
{
|
||||||
|
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||||
|
FailMutations = true
|
||||||
|
};
|
||||||
|
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var link = await links.CreateAsync(CreateRequest("target-one"), "operator", cancellationToken);
|
||||||
|
applier.EraseRules();
|
||||||
|
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z"));
|
||||||
|
var logger = new TestLogger<LinkReconciliationBackgroundService>();
|
||||||
|
var background = CreateBackgroundService(links, applier, time, logger);
|
||||||
|
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(applier.ReconciliationRequest);
|
||||||
|
var warning = Assert.Single(logger.Warnings, message => message.Contains("prompt exhausted", StringComparison.Ordinal));
|
||||||
|
Assert.Contains(link.Id, warning, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
time.Advance(TimeSpan.FromSeconds(30));
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.Single(logger.Warnings, message => message.Contains("prompt exhausted", StringComparison.Ordinal));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task MarkerCompletionFailuresAreAlsoThrottledAndMarkerIsRetained()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier
|
||||||
|
{
|
||||||
|
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||||
|
FailCompletion = true
|
||||||
|
};
|
||||||
|
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z"));
|
||||||
|
var background = CreateBackgroundService(links, applier, time);
|
||||||
|
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(applier.ReconciliationRequest);
|
||||||
|
Assert.Equal(3, applier.CompleteReconciliationCalls);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task GenericMarkerPassFailuresArePromptedThreeTimesThenUseRegularThrottle()
|
||||||
|
{
|
||||||
|
var cancellationToken = TestContext.Current.CancellationToken;
|
||||||
|
var store = CreateStore();
|
||||||
|
await store.InitializeAsync(cancellationToken);
|
||||||
|
var applier = new RecordingPolicyApplier
|
||||||
|
{
|
||||||
|
ReconciliationRequest = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||||
|
FailList = true
|
||||||
|
};
|
||||||
|
var links = new LinkService(store, applier, new ControlEventBroker());
|
||||||
|
var time = new TestTimeProvider(DateTimeOffset.Parse("2026-08-03T12:00:00Z"));
|
||||||
|
var background = CreateBackgroundService(links, applier, time);
|
||||||
|
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(() => background.RunOnceAsync(cancellationToken));
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(() => background.RunOnceAsync(cancellationToken));
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(() => background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.Null(await background.RunOnceAsync(cancellationToken));
|
||||||
|
Assert.NotNull(applier.ReconciliationRequest);
|
||||||
|
Assert.Equal(3, applier.ListCalls);
|
||||||
|
|
||||||
|
time.Advance(TimeSpan.FromSeconds(30));
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(() => background.RunOnceAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task BackgroundPassRetainsMarkerWhenFirewallIsUnavailable()
|
public async Task BackgroundPassRetainsMarkerWhenFirewallIsUnavailable()
|
||||||
{
|
{
|
||||||
|
|
@ -250,13 +731,14 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
private static LinkReconciliationBackgroundService CreateBackgroundService(
|
private static LinkReconciliationBackgroundService CreateBackgroundService(
|
||||||
LinkService links,
|
LinkService links,
|
||||||
ILinkPolicyApplier applier,
|
ILinkPolicyApplier applier,
|
||||||
TimeProvider timeProvider)
|
TimeProvider timeProvider,
|
||||||
|
ILogger<LinkReconciliationBackgroundService>? logger = null)
|
||||||
=> new(
|
=> new(
|
||||||
links,
|
links,
|
||||||
applier,
|
applier,
|
||||||
Options.Create(new ControlOptions { LinkReconciliationSeconds = 30 }),
|
Options.Create(new ControlOptions { LinkReconciliationSeconds = 30 }),
|
||||||
timeProvider,
|
timeProvider,
|
||||||
NullLogger<LinkReconciliationBackgroundService>.Instance);
|
logger ?? NullLogger<LinkReconciliationBackgroundService>.Instance);
|
||||||
|
|
||||||
private static async Task EnrollAgentAsync(
|
private static async Task EnrollAgentAsync(
|
||||||
ControlStore store,
|
ControlStore store,
|
||||||
|
|
@ -273,39 +755,130 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
|
|
||||||
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
private sealed class RecordingPolicyApplier : ILinkPolicyApplier
|
||||||
{
|
{
|
||||||
private readonly HashSet<string> _connected = [];
|
private readonly List<LinkRule> _rules = [];
|
||||||
|
private readonly Dictionary<LinkRule, int> _mutationCalls = [];
|
||||||
|
public TaskCompletionSource ListObserved { get; } = new(
|
||||||
|
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public TaskCompletionSource PausedListObserved { get; } = new(
|
||||||
|
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public TaskCompletionSource ReleasePausedList { get; } = new(
|
||||||
|
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
public int ListCalls { get; private set; }
|
||||||
|
public int? PauseOnListCall { get; set; }
|
||||||
public int ConnectCalls { get; private set; }
|
public int ConnectCalls { get; private set; }
|
||||||
public int DisconnectCalls { get; private set; }
|
public int DisconnectCalls { get; private set; }
|
||||||
|
public int StatusCalls { get; private set; }
|
||||||
|
public int PrivilegedCalls => ListCalls + ConnectCalls + DisconnectCalls + StatusCalls;
|
||||||
public bool FirewallUnavailable { get; set; }
|
public bool FirewallUnavailable { get; set; }
|
||||||
|
public bool FailMutations { get; set; }
|
||||||
|
public bool FailList { get; set; }
|
||||||
|
public bool UnavailableOnNextMutation { get; set; }
|
||||||
|
public bool UnavailableOnNextPostMutationList { get; set; }
|
||||||
|
public bool DisconnectLeavesRules { get; set; }
|
||||||
|
public bool ConnectAddsDuplicate { get; set; }
|
||||||
|
public bool ThrowNodeNotActivatedOnStatus { get; set; }
|
||||||
|
public HashSet<LinkRule> FailedMutationRules { get; } = [];
|
||||||
|
public HashSet<LinkRule> DeferredConnectRules { get; } = [];
|
||||||
public string? ReconciliationRequest { get; set; }
|
public string? ReconciliationRequest { get; set; }
|
||||||
public string? ReplacementRequestOnNextProbe { get; set; }
|
public string? ReplacementRequestOnNextProbe { get; set; }
|
||||||
public int ReconciliationStatusCalls { get; private set; }
|
public int ReconciliationStatusCalls { get; private set; }
|
||||||
public int CompleteReconciliationCalls { get; private set; }
|
public int CompleteReconciliationCalls { get; private set; }
|
||||||
|
public bool FailCompletion { get; set; }
|
||||||
|
public int ForeignRuleCount { get; set; }
|
||||||
public List<string> CompletedGenerations { get; } = [];
|
public List<string> CompletedGenerations { get; } = [];
|
||||||
|
|
||||||
public void EraseRules() => _connected.Clear();
|
public void EraseRules()
|
||||||
public bool IsConnected(LinkPolicy link) => _connected.Contains(link.Id);
|
{
|
||||||
|
_rules.Clear();
|
||||||
|
_mutationCalls.Clear();
|
||||||
|
}
|
||||||
|
public void InjectRule(LinkRule rule, int count = 1)
|
||||||
|
{
|
||||||
|
for (var index = 0; index < count; index++)
|
||||||
|
{
|
||||||
|
_rules.Add(rule);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public int RuleCount(LinkPolicy link) => _rules.Count(rule => rule == ToRule(link));
|
||||||
|
public int MutationCalls(LinkPolicy link)
|
||||||
|
=> _mutationCalls.GetValueOrDefault(ToRule(link));
|
||||||
|
public bool IsConnected(LinkPolicy link) => RuleCount(link) > 0;
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<LinkRule>> ListRulesAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
ListCalls++;
|
||||||
|
ListObserved.TrySetResult();
|
||||||
|
ReplaceRequestIfNeeded();
|
||||||
|
if (UnavailableOnNextPostMutationList && _mutationCalls.Count > 0)
|
||||||
|
{
|
||||||
|
UnavailableOnNextPostMutationList = false;
|
||||||
|
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
||||||
|
}
|
||||||
|
if (FailList)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("simulated link-list failure");
|
||||||
|
}
|
||||||
|
if (FirewallUnavailable)
|
||||||
|
{
|
||||||
|
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
||||||
|
}
|
||||||
|
var result = _rules.ToArray();
|
||||||
|
if (ListCalls == PauseOnListCall)
|
||||||
|
{
|
||||||
|
PausedListObserved.TrySetResult();
|
||||||
|
await ReleasePausedList.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyConnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
ConnectCalls++;
|
ConnectCalls++;
|
||||||
_connected.Add(link.Id);
|
var rule = ToRule(link);
|
||||||
|
RecordMutation(rule);
|
||||||
|
ThrowIfMutationUnavailable();
|
||||||
|
if (DeferredConnectRules.Contains(rule))
|
||||||
|
{
|
||||||
|
throw new MeshNodeNotActivatedException(LinkService.NodeNotActivatedCode);
|
||||||
|
}
|
||||||
|
if (FailMutations || FailedMutationRules.Contains(rule))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("simulated policy failure");
|
||||||
|
}
|
||||||
|
_rules.Add(rule);
|
||||||
|
if (ConnectAddsDuplicate)
|
||||||
|
{
|
||||||
|
_rules.Add(ToRule(link));
|
||||||
|
}
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task ApplyDisconnectAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
|
=> ApplyDisconnectAsync(ToRule(link), cancellationToken);
|
||||||
|
|
||||||
|
public Task ApplyDisconnectAsync(LinkRule rule, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
DisconnectCalls++;
|
DisconnectCalls++;
|
||||||
_connected.Remove(link.Id);
|
RecordMutation(rule);
|
||||||
|
ThrowIfMutationUnavailable();
|
||||||
|
if (FailMutations)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("simulated policy failure");
|
||||||
|
}
|
||||||
|
if (!DisconnectLeavesRules)
|
||||||
|
{
|
||||||
|
_rules.RemoveAll(candidate => candidate == rule);
|
||||||
|
}
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
public Task<bool> IsConnectedAsync(LinkPolicy link, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (ReplacementRequestOnNextProbe is not null)
|
StatusCalls++;
|
||||||
|
ReplaceRequestIfNeeded();
|
||||||
|
if (ThrowNodeNotActivatedOnStatus)
|
||||||
{
|
{
|
||||||
ReconciliationRequest = ReplacementRequestOnNextProbe;
|
throw new MeshNodeNotActivatedException(LinkService.NodeNotActivatedCode);
|
||||||
ReplacementRequestOnNextProbe = null;
|
|
||||||
}
|
}
|
||||||
return FirewallUnavailable
|
return FirewallUnavailable
|
||||||
? Task.FromException<bool>(new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode))
|
? Task.FromException<bool>(new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode))
|
||||||
|
|
@ -321,6 +894,10 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
public Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
|
public Task CompleteReconciliationAsync(string generation, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
CompleteReconciliationCalls++;
|
CompleteReconciliationCalls++;
|
||||||
|
if (FailCompletion)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("simulated marker completion failure");
|
||||||
|
}
|
||||||
CompletedGenerations.Add(generation);
|
CompletedGenerations.Add(generation);
|
||||||
if (ReconciliationRequest == generation)
|
if (ReconciliationRequest == generation)
|
||||||
{
|
{
|
||||||
|
|
@ -328,11 +905,60 @@ public sealed class LinkReconciliationTests : IAsyncDisposable
|
||||||
}
|
}
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void ReplaceRequestIfNeeded()
|
||||||
|
{
|
||||||
|
if (ReplacementRequestOnNextProbe is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ReconciliationRequest = ReplacementRequestOnNextProbe;
|
||||||
|
ReplacementRequestOnNextProbe = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void RecordMutation(LinkRule rule)
|
||||||
|
=> _mutationCalls[rule] = _mutationCalls.GetValueOrDefault(rule) + 1;
|
||||||
|
|
||||||
|
private void ThrowIfMutationUnavailable()
|
||||||
|
{
|
||||||
|
if (!UnavailableOnNextMutation)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
UnavailableOnNextMutation = false;
|
||||||
|
throw new MeshFirewallUnavailableException(LinkService.FirewallUnavailableCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static LinkRule ToRule(LinkPolicy link)
|
||||||
|
=> new(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static LinkRule ToRule(LinkPolicy link)
|
||||||
|
=> new(link.SourceNodeId, link.TargetNodeId, link.Protocol, link.Port);
|
||||||
|
|
||||||
private sealed class TestTimeProvider(DateTimeOffset now) : TimeProvider
|
private sealed class TestTimeProvider(DateTimeOffset now) : TimeProvider
|
||||||
{
|
{
|
||||||
public override DateTimeOffset GetUtcNow() => now;
|
public override DateTimeOffset GetUtcNow() => now;
|
||||||
public void Advance(TimeSpan value) => now += value;
|
public void Advance(TimeSpan value) => now += value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class TestLogger<T> : ILogger<T>
|
||||||
|
{
|
||||||
|
public List<string> Warnings { get; } = [];
|
||||||
|
|
||||||
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||||
|
public bool IsEnabled(LogLevel logLevel) => true;
|
||||||
|
public void Log<TState>(
|
||||||
|
LogLevel logLevel,
|
||||||
|
EventId eventId,
|
||||||
|
TState state,
|
||||||
|
Exception? exception,
|
||||||
|
Func<TState, Exception?, string> formatter)
|
||||||
|
{
|
||||||
|
if (logLevel == LogLevel.Warning)
|
||||||
|
{
|
||||||
|
Warnings.Add(formatter(state, exception));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -76,27 +76,27 @@ expect_blocked() {
|
||||||
}
|
}
|
||||||
|
|
||||||
probe_factual_status() {
|
probe_factual_status() {
|
||||||
local target="$1"
|
local target="$1"
|
||||||
local expected="$2"
|
local expected="$2"
|
||||||
local command output
|
local command output
|
||||||
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
||||||
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
||||||
output="$(hub_ssh "$command")"
|
output="$(hub_ssh "$command")"
|
||||||
[[ "$output" == "$expected" ]]
|
[[ "$output" == "$expected" ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
expect_factual_status() {
|
expect_factual_status() {
|
||||||
local target="$1"
|
local target="$1"
|
||||||
local expected="$2"
|
local expected="$2"
|
||||||
if ! probe_factual_status "$target" "$expected"; then
|
if ! probe_factual_status "$target" "$expected"; then
|
||||||
local command output
|
local command output
|
||||||
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
printf -v command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
||||||
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
link-status "$SOURCE_NODE_ID" "$target" tcp "$TARGET_PORT"
|
||||||
output="$(hub_ssh "$command")"
|
output="$(hub_ssh "$command")"
|
||||||
echo "Unexpected factual Link status for $SOURCE_NODE_ID -> $target: $output (expected $expected)" >&2
|
echo "Unexpected factual Link status for $SOURCE_NODE_ID -> $target: $output (expected $expected)" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
decode_base64url() {
|
decode_base64url() {
|
||||||
local value="${1//-/+}"
|
local value="${1//-/+}"
|
||||||
|
|
@ -237,6 +237,21 @@ if [[ "${SMM_ACCEPT_RESTORE:-0}" == '1' ]]; then
|
||||||
expect_factual_status "$SECOND_NODE_ID" disabled
|
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||||
expect_reachable "$HOME_WG_IP"
|
expect_reachable "$HOME_WG_IP"
|
||||||
expect_blocked "$SECOND_WG_IP"
|
expect_blocked "$SECOND_WG_IP"
|
||||||
|
|
||||||
|
echo '[8/12] Injecting an orphan accept rule for the disabled policy'
|
||||||
|
printf -v orphan_command '%q ' sudo /usr/local/libexec/ochenstarik-smm-policy-apply \
|
||||||
|
link-connect "$SOURCE_NODE_ID" "$SECOND_NODE_ID" tcp "$TARGET_PORT" 0
|
||||||
|
hub_ssh "$orphan_command" >/dev/null
|
||||||
|
expect_factual_status "$SECOND_NODE_ID" active
|
||||||
|
deadline=$((SECONDS + LINK_RECONCILIATION_SECONDS + 30))
|
||||||
|
while ((SECONDS < deadline)); do
|
||||||
|
if probe_factual_status "$SECOND_NODE_ID" disabled && probe_blocked "$SECOND_WG_IP"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
expect_factual_status "$SECOND_NODE_ID" disabled
|
||||||
|
expect_blocked "$SECOND_WG_IP"
|
||||||
else
|
else
|
||||||
echo '[8/12] Firewall restore reconciliation skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
|
echo '[8/12] Firewall restore reconciliation skipped; set SMM_ACCEPT_RESTORE=1 to enable it'
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,90 @@ if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$policy_state" \
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
policy_listing="$(mktemp -t smm-policy-listing.XXXXXXXX)"
|
||||||
|
cat >"$policy_listing" <<'EOF'
|
||||||
|
ip saddr 10.77.0.2 ip daddr 10.77.0.3 tcp dport 22 counter accept comment "smm:source:target:tcp:22" # handle 5
|
||||||
|
counter accept comment "foreign:keep-me" # handle 6
|
||||||
|
counter accept comment "smm:source:target:tcp:22" # handle 7
|
||||||
|
counter accept comment "smm:FORGED:target:tcp:22" # handle 8
|
||||||
|
EOF
|
||||||
|
list_error="$(mktemp -t smm-policy-list-error.XXXXXXXX)"
|
||||||
|
list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$policy_listing" \
|
||||||
|
bash "$helper" link-list 2>"$list_error")"
|
||||||
|
[[ "$list_output" == $'source\ttarget\ttcp\t22\nsource\ttarget\ttcp\t22' ]]
|
||||||
|
if grep -Fq 'foreign:keep-me' <<<"$list_output"; then
|
||||||
|
printf '%s\n' "policy helper exposed a foreign nftables comment" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
grep -Fq 'forged managed comment ignored' "$list_error"
|
||||||
|
empty_listing="$(mktemp -t smm-policy-empty-listing.XXXXXXXX)"
|
||||||
|
empty_list_output="$(SMM_POLICY_TESTING=1 SMM_POLICY_LISTING_FILE="$empty_listing" \
|
||||||
|
bash "$helper" link-list)"
|
||||||
|
[[ -z "$empty_list_output" ]]
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_FIREWALL_UNAVAILABLE=1 \
|
||||||
|
bash "$helper" link-list >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "missing Link policy table unexpectedly produced a listing" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 79 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'mesh.firewall-unavailable' ]]
|
||||||
|
if SMM_POLICY_TESTING=1 bash "$helper" link-list unexpected >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly accepted extra link-list arguments" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
inactive_state="$(mktemp -t smm-policy-inactive-state.XXXXXXXX)"
|
||||||
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
||||||
|
printf 'target\t10.77.0.3\t-\treserved\n' >>"$inactive_state"
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
||||||
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly activated a Link to a reserved Node" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 80 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'mesh.node-not-activated' ]]
|
||||||
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
||||||
|
printf 'target\t\tkey-target\tactive\n' >>"$inactive_state"
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
||||||
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly accepted a blank Node address" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 78 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
||||||
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
||||||
|
printf 'target\tnot-an-ip\tkey-target\tactive\n' >>"$inactive_state"
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
||||||
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly accepted an invalid Node address" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 78 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
||||||
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
||||||
|
printf 'target\t999.77.0.3\tkey-target\tactive\n' >>"$inactive_state"
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
||||||
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly accepted an out-of-range Node address" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 78 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'policy helper: node has no valid mesh address: target' ]]
|
||||||
|
printf 'source\t10.77.0.2\tkey-source\tactive\n' >"$inactive_state"
|
||||||
|
printf 'target\t10.77.0.3\tkey-target\tgarbage!\n' >>"$inactive_state"
|
||||||
|
if SMM_POLICY_TESTING=1 SMM_POLICY_STATE_FILE="$inactive_state" \
|
||||||
|
bash "$helper" link-connect source target tcp 22 10 >/dev/null 2>"$list_error"; then
|
||||||
|
printf '%s\n' "policy helper unexpectedly accepted a malformed Node status" >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
[[ $? -eq 78 ]]
|
||||||
|
fi
|
||||||
|
[[ "$(<"$list_error")" == 'policy helper: invalid mesh node status: target' ]]
|
||||||
|
|
||||||
generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
generation_a='aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||||
generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
|
generation_b='bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
|
||||||
reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)"
|
reconcile_marker="$(mktemp -t smm-reconcile-marker.XXXXXXXX)"
|
||||||
|
|
@ -122,6 +206,9 @@ SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconci
|
||||||
[[ ! -e "$reconcile_marker" ]]
|
[[ ! -e "$reconcile_marker" ]]
|
||||||
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||||
bash "$helper" reconcile-status)" == 'complete' ]]
|
bash "$helper" reconcile-status)" == 'complete' ]]
|
||||||
|
missing_mesh="$reconcile_marker-missing/mesh/reconcile-requested"
|
||||||
|
[[ "$(SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$missing_mesh" \
|
||||||
|
bash "$helper" reconcile-status)" == 'complete' ]]
|
||||||
if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
if SMM_POLICY_TESTING=1 SMM_POLICY_FLOCK=true SMM_POLICY_RECONCILE_MARKER="$reconcile_marker" \
|
||||||
bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then
|
bash "$helper" reconcile-complete unexpected extra >/dev/null 2>&1; then
|
||||||
printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2
|
printf '%s\n' "policy helper unexpectedly accepted extra reconcile-complete arguments" >&2
|
||||||
|
|
@ -151,8 +238,8 @@ if grep -Fq 'mesh.firewall-unavailable' "$firewall_error"; then
|
||||||
printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2
|
printf '%s\n' "unknown nft inspection error was misclassified as missing firewall" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
rm -f -- "$firewall_error" "$reconcile_marker"
|
rm -f -- "$firewall_error" "$reconcile_marker" "$policy_state" \
|
||||||
rm -f -- "$policy_state"
|
"$policy_listing" "$empty_listing" "$list_error" "$inactive_state"
|
||||||
|
|
||||||
extract_shell_function() {
|
extract_shell_function() {
|
||||||
local name="$1"
|
local name="$1"
|
||||||
|
|
|
||||||
|
|
@ -62,9 +62,36 @@ if ($linksCode.IndexOf('SetFirewallUnavailable', [StringComparison]::Ordinal) -l
|
||||||
$mainCode.IndexOf('MeshLinkViewModel.FirewallUnavailableErrorCode', [StringComparison]::Ordinal) -lt 0) {
|
$mainCode.IndexOf('MeshLinkViewModel.FirewallUnavailableErrorCode', [StringComparison]::Ordinal) -lt 0) {
|
||||||
throw 'Desktop must project both event and persisted Mesh firewall unavailable state.'
|
throw 'Desktop must project both event and persisted Mesh firewall unavailable state.'
|
||||||
}
|
}
|
||||||
if ($meshModelsCode.IndexOf('LastError == FirewallUnavailableErrorCode ? string.Empty',
|
if ($meshModelsCode.IndexOf(
|
||||||
|
'LastError is FirewallUnavailableErrorCode or NodeNotActivatedErrorCode ? string.Empty',
|
||||||
[StringComparison]::Ordinal) -lt 0) {
|
[StringComparison]::Ordinal) -lt 0) {
|
||||||
throw 'Shared Mesh firewall errors must be suppressed from individual Link rows.'
|
throw 'Shared Mesh firewall and expected Node activation states must be suppressed from individual Link errors.'
|
||||||
|
}
|
||||||
|
if ($linksXaml.IndexOf('x:Name="ShowHistoryToggle"', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$linksXaml.IndexOf('Toggled="ShowHistoryToggle_Toggled"', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$linksCode.IndexOf('ShowHistoryToggle.IsOn', [StringComparison]::Ordinal) -lt 0) {
|
||||||
|
throw 'Links page must expose an explicit history toggle.'
|
||||||
|
}
|
||||||
|
if ($linksCode.IndexOf(
|
||||||
|
'effective.Where(link => link.DesiredState == "Active" || link.HasDrift)',
|
||||||
|
[StringComparison]::Ordinal) -lt 0) {
|
||||||
|
throw 'Links page default view must show only effective Active or drifted policies.'
|
||||||
|
}
|
||||||
|
if ($linksCode.IndexOf('LinksCountText.Text = $', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$linksCode.IndexOf('DisplayedLinks.Count(link => link.ActualState == "Active")', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$linksCode.IndexOf('DisplayedLinks.Count(link => link.HasDrift)', [StringComparison]::Ordinal) -lt 0) {
|
||||||
|
throw 'Links page counters must unambiguously distinguish shown, factual Active, and drifted policies.'
|
||||||
|
}
|
||||||
|
if ($meshModelsCode.IndexOf('public string DesiredStatusText', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$meshModelsCode.IndexOf('public string ActualStatusText', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$meshModelsCode.IndexOf('LastError == NodeNotActivatedErrorCode', [StringComparison]::Ordinal) -lt 0) {
|
||||||
|
throw 'Links page must use typed desired/factual/expected-activation wording.'
|
||||||
|
}
|
||||||
|
if ($mainCode.IndexOf(
|
||||||
|
'link.LastError == MeshLinkViewModel.NodeNotActivatedErrorCode',
|
||||||
|
[StringComparison]::Ordinal) -lt 0 -or
|
||||||
|
$mainCode.IndexOf('? InfoBarSeverity.Informational', [StringComparison]::Ordinal) -lt 0) {
|
||||||
|
throw 'Expected Node activation state must use non-error informational severity.'
|
||||||
}
|
}
|
||||||
if ($mainCode.IndexOf('Servers.Count > 0 || _control.IsConfigured', [StringComparison]::Ordinal) -lt 0 -or
|
if ($mainCode.IndexOf('Servers.Count > 0 || _control.IsConfigured', [StringComparison]::Ordinal) -lt 0 -or
|
||||||
$mainCode.IndexOf('await RefreshControlMeshAsync(showSuccess: false);', [StringComparison]::Ordinal) -lt 0) {
|
$mainCode.IndexOf('await RefreshControlMeshAsync(showSuccess: false);', [StringComparison]::Ordinal) -lt 0) {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue