Add persistent mTLS control layer
This commit is contained in:
parent
7bd7c950a6
commit
46a1c393c7
28 changed files with 1360 additions and 3 deletions
39
.github/workflows/linux-control-agent.yml
vendored
Normal file
39
.github/workflows/linux-control-agent.yml
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
name: Linux control and agent
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ['**']
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up .NET 10
|
||||
uses: actions/setup-dotnet@v5
|
||||
with:
|
||||
dotnet-version: 10.0.x
|
||||
|
||||
- name: Restore
|
||||
run: dotnet restore ServerMonitorManager.slnx
|
||||
|
||||
- name: Build
|
||||
run: dotnet build ServerMonitorManager.slnx --configuration Release --no-restore
|
||||
|
||||
- name: Test
|
||||
run: dotnet test tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj --configuration Release --no-build
|
||||
|
||||
- name: Verify formatting
|
||||
run: dotnet format ServerMonitorManager.slnx --verify-no-changes --no-restore
|
||||
|
||||
- name: Publish agent amd64
|
||||
run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-x64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
|
||||
|
||||
- name: Publish agent arm64
|
||||
run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime linux-arm64 --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true
|
||||
55
.github/workflows/linux-release.yml
vendored
Normal file
55
.github/workflows/linux-release.yml
vendored
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
name: Linux release artifacts
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
runtime: [linux-x64, linux-arm64]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up .NET 10
|
||||
uses: actions/setup-dotnet@v5
|
||||
with:
|
||||
dotnet-version: 10.0.x
|
||||
|
||||
- name: Publish agent
|
||||
run: dotnet publish src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/agent
|
||||
|
||||
- name: Publish control
|
||||
run: dotnet publish src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj --configuration Release --runtime ${{ matrix.runtime }} --self-contained true -p:PublishSingleFile=true -p:PublishTrimmed=true -o out/control
|
||||
|
||||
- name: Package
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
archive="server-monitor-manager-${{ matrix.runtime }}.tar.gz"
|
||||
tar -C out -czf "$archive" agent control
|
||||
sha256sum "$archive" > "$archive.sha256"
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: server-monitor-manager-${{ matrix.runtime }}
|
||||
path: |
|
||||
server-monitor-manager-${{ matrix.runtime }}.tar.gz
|
||||
server-monitor-manager-${{ matrix.runtime }}.tar.gz.sha256
|
||||
|
||||
- name: Attach to GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
server-monitor-manager-${{ matrix.runtime }}.tar.gz
|
||||
server-monitor-manager-${{ matrix.runtime }}.tar.gz.sha256
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -4,6 +4,7 @@
|
|||
bin/
|
||||
obj/
|
||||
artifacts/
|
||||
.artifacts/
|
||||
TestResults/
|
||||
*.user
|
||||
*.suo
|
||||
|
|
|
|||
22
README.md
22
README.md
|
|
@ -45,6 +45,28 @@ tests/
|
|||
|
||||
Установочный скрипт Linux-части хранится в репозитории [`ochenstarik-ui/lightweight-server`](https://github.com/ochenstarik-ui/lightweight-server) под именем `ochenstarik-server-monitor-manager.sh`. Он устанавливает режим Hub или Node, создаёт отдельного пользователя `ochenstarik-monitor` и SSH forced-command: ключ приложения не получает shell, PTY, port forwarding или право выполнять произвольные команды.
|
||||
|
||||
В разработческой ветке также появился первый срез постоянного control layer: ASP.NET Core 10 Hub, SQLite и исходящий Linux Agent. Регистрация использует одноразовый token и CSR, дальнейшие heartbeat-запросы — mTLS. Этот слой ещё не заменяет проверенный SSH/WireGuard установщик: сначала будут добавлены release-бинарники, systemd-установка и миграция Links.
|
||||
|
||||
Alpha-установка постоянного слоя после обычных ролей Hub/Node:
|
||||
|
||||
```bash
|
||||
# Hub
|
||||
sudo ./ochenstarik-server-monitor-manager.sh install-control-hub
|
||||
sudo ./ochenstarik-server-monitor-manager.sh control-code home
|
||||
|
||||
# соответствующий Node — вставить полученный SMMCTL1
|
||||
sudo ./ochenstarik-server-monitor-manager.sh install-control-agent
|
||||
```
|
||||
|
||||
Архив выбирается автоматически для amd64 или arm64 и проверяется по SHA-256. Для Control Hub требуется входящий TCP-порт `7443`; Agent открытых входящих портов не создаёт.
|
||||
|
||||
Проверка control layer для разработчиков:
|
||||
|
||||
```bash
|
||||
dotnet build ServerMonitorManager.slnx --configuration Release
|
||||
dotnet test tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj --configuration Release
|
||||
```
|
||||
|
||||
## Быстрый тест на трёх серверах
|
||||
|
||||
1. Запустите Windows-клиент и нажмите `SSH-ключ` → `Копировать`.
|
||||
|
|
|
|||
6
ServerMonitorManager.slnx
Normal file
6
ServerMonitorManager.slnx
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
<Solution>
|
||||
<Project Path="src/ServerMonitorManager.Agent/ServerMonitorManager.Agent.csproj" />
|
||||
<Project Path="src/ServerMonitorManager.Control/ServerMonitorManager.Control.csproj" />
|
||||
<Project Path="src/ServerMonitorManager.Core/ServerMonitorManager.Core.csproj" />
|
||||
<Project Path="tests/ServerMonitorManager.Control.Tests/ServerMonitorManager.Control.Tests.csproj" />
|
||||
</Solution>
|
||||
28
deploy/ochenstarik-smm-agent.service
Normal file
28
deploy/ochenstarik-smm-agent.service
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=Ochenstarik Server Monitor Manager Agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ochenstarik-smm-agent
|
||||
Group=ochenstarik-smm-agent
|
||||
EnvironmentFile=/etc/ochenstarik-server-monitor-manager/agent.env
|
||||
ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/agent/ochenstarik-smm-agent
|
||||
Restart=on-failure
|
||||
RestartSec=10s
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
LockPersonality=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager/agent
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
28
deploy/ochenstarik-smm-control.service
Normal file
28
deploy/ochenstarik-smm-control.service
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=Ochenstarik Server Monitor Manager Control Hub
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ochenstarik-smm-control
|
||||
Group=ochenstarik-smm-control
|
||||
EnvironmentFile=/etc/ochenstarik-server-monitor-manager/control.env
|
||||
ExecStart=/usr/local/lib/ochenstarik-server-monitor-manager/control/ochenstarik-smm-control
|
||||
Restart=on-failure
|
||||
RestartSec=10s
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
LockPersonality=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
ReadWritePaths=/var/lib/ochenstarik-server-monitor-manager
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -96,6 +96,10 @@ Active -> Expired -> Disabled
|
|||
|
||||
Hub остаётся маршрутизатором Mesh первого поколения. Разделение control plane и data plane возможно позже без изменения модели направленных Links.
|
||||
|
||||
Первый реализованный срез control layer использует ASP.NET Core 10 и SQLite. Hub выдаёт агенту сертификат по CSR только после атомарного погашения десятиминутного token. После регистрации Agent выполняет только исходящие HTTPS-запросы с mTLS, а Hub связывает thumbprint сертификата с конкретным `node_id`. Heartbeat содержит idempotency key и отклоняется при попытке повторить тот же ключ с другим телом запроса.
|
||||
|
||||
Сейчас этот срез сохраняет inventory, heartbeat-метрики, idempotency и аудит регистрации. Перенос политик Links из файлов в SQLite, постоянный event stream и локальный буфер Agent выполняются следующими частями этапа.
|
||||
|
||||
## 7. Целевые ограничения MVP
|
||||
|
||||
- без Kubernetes и обязательного Docker;
|
||||
|
|
|
|||
|
|
@ -38,6 +38,16 @@
|
|||
- создаёт только исходящее WireGuard-соединение;
|
||||
- не требует белого IP или входящего публичного порта.
|
||||
|
||||
## Постоянный control layer (alpha)
|
||||
|
||||
После установки существующих ролей отдельные действия добавляют постоянные сервисы:
|
||||
|
||||
- `install-control-hub` скачивает release-архив под amd64/arm64, проверяет SHA-256, создаёт локальный CA, HTTPS-сертификат Hub, SQLite-каталог и изолированный systemd service;
|
||||
- `control-code NAME` создаёт десятиминутный token и код `SMMCTL1`, содержащий URL Hub и только публичный CA;
|
||||
- `install-control-agent` проверяет CA, локально создаёт ключ и CSR, регистрирует сертификат и запускает исходящий mTLS Agent через systemd.
|
||||
|
||||
Приватный ключ Control CA не включается в `SMMCTL1`, а приватный ключ Agent не покидает Node. По умолчанию Control Hub слушает TCP `7443`.
|
||||
|
||||
## Команды жизненного цикла
|
||||
|
||||
Целевой интерфейс:
|
||||
|
|
@ -46,6 +56,9 @@
|
|||
install-monitor
|
||||
install-hub
|
||||
install-node
|
||||
install-control-hub
|
||||
install-control-agent
|
||||
control-code NAME
|
||||
status
|
||||
update
|
||||
rollback
|
||||
|
|
|
|||
|
|
@ -70,12 +70,12 @@
|
|||
|
||||
## Этап 6 — постоянный control layer
|
||||
|
||||
- [ ] статический Linux agent для amd64/arm64;
|
||||
- [x] самодостаточный single-file Linux agent для amd64/arm64;
|
||||
- [ ] SQLite inventory, policies, history и audit;
|
||||
- [ ] исходящие mTLS agent sessions;
|
||||
- [x] исходящие mTLS agent sessions;
|
||||
- [ ] WebSocket/stream событий для desktop client;
|
||||
- [ ] ограниченный локальный буфер и downsampling;
|
||||
- [ ] idempotency key и защита от replay;
|
||||
- [x] idempotency key и защита от replay;
|
||||
- [ ] тест нагрузки 50–100 Node на одном Hub.
|
||||
|
||||
## Этап 7 — релиз и другие платформы
|
||||
|
|
|
|||
105
src/ServerMonitorManager.Agent/AgentClient.cs
Normal file
105
src/ServerMonitorManager.Agent/AgentClient.cs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using ServerMonitorManager.Core;
|
||||
|
||||
namespace ServerMonitorManager.Agent;
|
||||
|
||||
internal sealed class AgentClient(AgentOptions options)
|
||||
{
|
||||
private readonly string _certificatePath = Path.Combine(options.StateDirectory, "agent.pfx");
|
||||
|
||||
public async Task EnrollAsync(string token, CancellationToken cancellationToken)
|
||||
{
|
||||
Directory.CreateDirectory(options.StateDirectory);
|
||||
using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var request = new CertificateRequest(
|
||||
$"CN={options.NodeId}",
|
||||
key,
|
||||
HashAlgorithmName.SHA256);
|
||||
var enrollment = new EnrollmentRequest(
|
||||
options.NodeId,
|
||||
token,
|
||||
request.CreateSigningRequestPem(),
|
||||
Guid.NewGuid().ToString());
|
||||
|
||||
using var client = CreateHttpClient(clientCertificate: null);
|
||||
using var response = await client.PostAsJsonAsync(
|
||||
"api/v1/enroll",
|
||||
enrollment,
|
||||
SmmJsonContext.Default.EnrollmentRequest,
|
||||
cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
var result = await response.Content.ReadFromJsonAsync(
|
||||
SmmJsonContext.Default.EnrollmentResponse,
|
||||
cancellationToken)
|
||||
?? throw new InvalidOperationException("Control service returned an empty enrollment response.");
|
||||
using var certificate = X509Certificate2.CreateFromPem(result.CertificatePem, key.ExportPkcs8PrivateKeyPem());
|
||||
await File.WriteAllBytesAsync(_certificatePath, certificate.Export(X509ContentType.Pfx), cancellationToken);
|
||||
await File.WriteAllTextAsync(options.CertificateAuthorityPath, result.CertificateAuthorityPem, cancellationToken);
|
||||
SetOwnerOnlyPermissions(_certificatePath);
|
||||
SetOwnerOnlyPermissions(options.CertificateAuthorityPath);
|
||||
}
|
||||
|
||||
public async Task RunAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var certificate = X509CertificateLoader.LoadPkcs12FromFile(
|
||||
_certificatePath,
|
||||
password: null,
|
||||
X509KeyStorageFlags.EphemeralKeySet);
|
||||
using var client = CreateHttpClient(certificate);
|
||||
var delay = TimeSpan.FromSeconds(options.HeartbeatSeconds);
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var heartbeat = LinuxMetrics.Collect(options.NodeId, "0.1.0");
|
||||
using var response = await client.PostAsJsonAsync(
|
||||
"api/v1/agents/heartbeat",
|
||||
heartbeat,
|
||||
SmmJsonContext.Default.AgentHeartbeat,
|
||||
cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
var accepted = await response.Content.ReadFromJsonAsync(
|
||||
SmmJsonContext.Default.AgentHeartbeatResponse,
|
||||
cancellationToken)
|
||||
?? throw new InvalidOperationException("Control service returned an empty heartbeat response.");
|
||||
delay = TimeSpan.FromSeconds(Math.Clamp(accepted.NextHeartbeatSeconds, 10, 300));
|
||||
await Task.Delay(delay, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpClient CreateHttpClient(X509Certificate2? clientCertificate)
|
||||
{
|
||||
using var root = X509CertificateLoader.LoadCertificateFromFile(options.CertificateAuthorityPath);
|
||||
var rootBytes = root.Export(X509ContentType.Cert);
|
||||
var handler = new HttpClientHandler();
|
||||
if (clientCertificate is not null)
|
||||
{
|
||||
handler.ClientCertificates.Add(clientCertificate);
|
||||
}
|
||||
handler.ServerCertificateCustomValidationCallback = (_, certificate, _, errors) =>
|
||||
{
|
||||
if (certificate is null
|
||||
|| errors.HasFlag(System.Net.Security.SslPolicyErrors.RemoteCertificateNameMismatch)
|
||||
|| errors.HasFlag(System.Net.Security.SslPolicyErrors.RemoteCertificateNotAvailable))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
using var trustedRoot = X509CertificateLoader.LoadCertificate(rootBytes);
|
||||
using var chain = new X509Chain();
|
||||
chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
|
||||
chain.ChainPolicy.CustomTrustStore.Add(trustedRoot);
|
||||
chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
|
||||
chain.ChainPolicy.ApplicationPolicy.Add(new Oid("1.3.6.1.5.5.7.3.1"));
|
||||
return chain.Build(new X509Certificate2(certificate));
|
||||
};
|
||||
return new HttpClient(handler) { BaseAddress = options.ControlUrl };
|
||||
}
|
||||
|
||||
private static void SetOwnerOnlyPermissions(string path)
|
||||
{
|
||||
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
|
||||
{
|
||||
File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite);
|
||||
}
|
||||
}
|
||||
}
|
||||
10
src/ServerMonitorManager.Agent/AgentOptions.cs
Normal file
10
src/ServerMonitorManager.Agent/AgentOptions.cs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
namespace ServerMonitorManager.Agent;
|
||||
|
||||
public sealed class AgentOptions
|
||||
{
|
||||
public string NodeId { get; init; } = Environment.MachineName.ToLowerInvariant();
|
||||
public Uri ControlUrl { get; init; } = new("https://127.0.0.1:7443");
|
||||
public string StateDirectory { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/agent";
|
||||
public string CertificateAuthorityPath { get; init; } = "/etc/ochenstarik-server-monitor-manager/control-ca.crt";
|
||||
public int HeartbeatSeconds { get; init; } = 30;
|
||||
}
|
||||
78
src/ServerMonitorManager.Agent/LinuxMetrics.cs
Normal file
78
src/ServerMonitorManager.Agent/LinuxMetrics.cs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
using ServerMonitorManager.Core;
|
||||
|
||||
namespace ServerMonitorManager.Agent;
|
||||
|
||||
internal static class LinuxMetrics
|
||||
{
|
||||
public static AgentHeartbeat Collect(string nodeId, string version)
|
||||
{
|
||||
var memory = ReadMemory();
|
||||
var disk = new DriveInfo("/");
|
||||
var network = ReadNetwork();
|
||||
return new AgentHeartbeat(
|
||||
nodeId,
|
||||
version,
|
||||
DateTimeOffset.UtcNow,
|
||||
ReadLoadOne(),
|
||||
memory.Total - memory.Available,
|
||||
memory.Total,
|
||||
disk.TotalSize - disk.AvailableFreeSpace,
|
||||
disk.TotalSize,
|
||||
network.Receive,
|
||||
network.Transmit,
|
||||
ReadUptimeSeconds(),
|
||||
Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
private static double ReadLoadOne()
|
||||
{
|
||||
var value = File.ReadAllText("/proc/loadavg").Split(' ', 2)[0];
|
||||
return double.Parse(value, System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static long ReadUptimeSeconds()
|
||||
{
|
||||
var value = File.ReadAllText("/proc/uptime").Split(' ', 2)[0];
|
||||
return (long)double.Parse(value, System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static (long Total, long Available) ReadMemory()
|
||||
{
|
||||
long total = 0;
|
||||
long available = 0;
|
||||
foreach (var line in File.ReadLines("/proc/meminfo"))
|
||||
{
|
||||
var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length < 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if (parts[0] == "MemTotal:")
|
||||
{
|
||||
total = long.Parse(parts[1]) * 1024;
|
||||
}
|
||||
else if (parts[0] == "MemAvailable:")
|
||||
{
|
||||
available = long.Parse(parts[1]) * 1024;
|
||||
}
|
||||
}
|
||||
return (total, available);
|
||||
}
|
||||
|
||||
private static (long Receive, long Transmit) ReadNetwork()
|
||||
{
|
||||
long receive = 0;
|
||||
long transmit = 0;
|
||||
foreach (var line in File.ReadLines("/proc/net/dev").Skip(2))
|
||||
{
|
||||
var parts = line.Split([':', ' '], StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length < 10 || parts[0] == "lo")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
receive += long.Parse(parts[1]);
|
||||
transmit += long.Parse(parts[9]);
|
||||
}
|
||||
return (receive, transmit);
|
||||
}
|
||||
}
|
||||
33
src/ServerMonitorManager.Agent/Program.cs
Normal file
33
src/ServerMonitorManager.Agent/Program.cs
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
using Microsoft.Extensions.Configuration;
|
||||
using ServerMonitorManager.Agent;
|
||||
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables("SMM_")
|
||||
.AddCommandLine(args)
|
||||
.Build();
|
||||
var options = configuration.Get<AgentOptions>() ?? new AgentOptions();
|
||||
if (string.IsNullOrWhiteSpace(options.NodeId)
|
||||
|| !options.NodeId.All(character => character is >= 'a' and <= 'z' or >= '0' and <= '9' or '-'))
|
||||
{
|
||||
Console.Error.WriteLine("NodeId must contain lowercase letters, digits, or hyphens.");
|
||||
return 2;
|
||||
}
|
||||
|
||||
using var shutdown = new CancellationTokenSource();
|
||||
Console.CancelKeyPress += (_, eventArgs) =>
|
||||
{
|
||||
eventArgs.Cancel = true;
|
||||
shutdown.Cancel();
|
||||
};
|
||||
var client = new AgentClient(options);
|
||||
var enrollmentToken = configuration["EnrollToken"];
|
||||
if (!string.IsNullOrWhiteSpace(enrollmentToken))
|
||||
{
|
||||
await client.EnrollAsync(enrollmentToken, shutdown.Token);
|
||||
Console.WriteLine("Agent enrollment completed.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
await client.RunAsync(shutdown.Token);
|
||||
return 0;
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
<Version>0.1.0</Version>
|
||||
<AssemblyName>ochenstarik-smm-agent</AssemblyName>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ServerMonitorManager.Core\ServerMonitorManager.Core.csproj" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.10" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
7
src/ServerMonitorManager.Agent/appsettings.json
Normal file
7
src/ServerMonitorManager.Agent/appsettings.json
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"NodeId": "change-me",
|
||||
"ControlUrl": "https://127.0.0.1:7443",
|
||||
"StateDirectory": "/var/lib/ochenstarik-server-monitor-manager/agent",
|
||||
"CertificateAuthorityPath": "/etc/ochenstarik-server-monitor-manager/control-ca.crt",
|
||||
"HeartbeatSeconds": 30
|
||||
}
|
||||
75
src/ServerMonitorManager.Control/CertificateAuthority.cs
Normal file
75
src/ServerMonitorManager.Control/CertificateAuthority.cs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ServerMonitorManager.Control;
|
||||
|
||||
public sealed class CertificateAuthority : IDisposable
|
||||
{
|
||||
private readonly X509Certificate2 _issuer;
|
||||
|
||||
public CertificateAuthority(IOptions<ControlOptions> options)
|
||||
{
|
||||
var value = options.Value;
|
||||
_issuer = X509CertificateLoader.LoadPkcs12FromFile(
|
||||
value.CertificateAuthorityPath,
|
||||
value.CertificateAuthorityPassword,
|
||||
X509KeyStorageFlags.EphemeralKeySet);
|
||||
if (!_issuer.HasPrivateKey)
|
||||
{
|
||||
throw new InvalidOperationException("Control CA certificate must contain its private key.");
|
||||
}
|
||||
}
|
||||
|
||||
public X509Certificate2 PublicCertificate => _issuer;
|
||||
|
||||
public IssuedCertificate IssueClientCertificate(string nodeId, string csrPem)
|
||||
{
|
||||
var request = CertificateRequest.LoadSigningRequestPem(
|
||||
csrPem,
|
||||
HashAlgorithmName.SHA256,
|
||||
CertificateRequestLoadOptions.UnsafeLoadCertificateExtensions);
|
||||
if (!string.Equals(request.SubjectName.Name, $"CN={nodeId}", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("CSR subject does not match the requested node id.");
|
||||
}
|
||||
if (request.CertificateExtensions.Count != 0)
|
||||
{
|
||||
throw new InvalidOperationException("CSR extensions are not accepted.");
|
||||
}
|
||||
|
||||
request.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, true));
|
||||
request.CertificateExtensions.Add(new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature, true));
|
||||
request.CertificateExtensions.Add(new X509EnhancedKeyUsageExtension(
|
||||
new OidCollection { new("1.3.6.1.5.5.7.3.2") },
|
||||
true));
|
||||
request.CertificateExtensions.Add(new X509SubjectKeyIdentifierExtension(request.PublicKey, false));
|
||||
|
||||
var serial = RandomNumberGenerator.GetBytes(20);
|
||||
var issuerNotBefore = new DateTimeOffset(_issuer.NotBefore.ToUniversalTime(), TimeSpan.Zero);
|
||||
var issuerNotAfter = new DateTimeOffset(_issuer.NotAfter.ToUniversalTime(), TimeSpan.Zero);
|
||||
var notBefore = DateTimeOffset.UtcNow.AddMinutes(-2) > issuerNotBefore
|
||||
? DateTimeOffset.UtcNow.AddMinutes(-2)
|
||||
: issuerNotBefore;
|
||||
var requestedNotAfter = DateTimeOffset.UtcNow.AddYears(1);
|
||||
var notAfter = requestedNotAfter < issuerNotAfter ? requestedNotAfter : issuerNotAfter;
|
||||
if (notAfter <= notBefore)
|
||||
{
|
||||
throw new InvalidOperationException("Control CA certificate is expired or not yet valid.");
|
||||
}
|
||||
using var certificate = request.Create(_issuer, notBefore, notAfter, serial);
|
||||
return new IssuedCertificate(
|
||||
certificate.ExportCertificatePem(),
|
||||
_issuer.ExportCertificatePem(),
|
||||
certificate.Thumbprint,
|
||||
notAfter);
|
||||
}
|
||||
|
||||
public void Dispose() => _issuer.Dispose();
|
||||
}
|
||||
|
||||
public sealed record IssuedCertificate(
|
||||
string CertificatePem,
|
||||
string CertificateAuthorityPem,
|
||||
string Thumbprint,
|
||||
DateTimeOffset ExpiresAt);
|
||||
14
src/ServerMonitorManager.Control/ControlOptions.cs
Normal file
14
src/ServerMonitorManager.Control/ControlOptions.cs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
namespace ServerMonitorManager.Control;
|
||||
|
||||
public sealed class ControlOptions
|
||||
{
|
||||
public const string SectionName = "Control";
|
||||
|
||||
public string DatabasePath { get; init; } = "/var/lib/ochenstarik-server-monitor-manager/control.db";
|
||||
|
||||
public string CertificateAuthorityPath { get; init; } = "/etc/ochenstarik-server-monitor-manager/control-ca.pfx";
|
||||
|
||||
public string? CertificateAuthorityPassword { get; init; }
|
||||
|
||||
public int HeartbeatSeconds { get; init; } = 30;
|
||||
}
|
||||
354
src/ServerMonitorManager.Control/ControlStore.cs
Normal file
354
src/ServerMonitorManager.Control/ControlStore.cs
Normal file
|
|
@ -0,0 +1,354 @@
|
|||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ServerMonitorManager.Core;
|
||||
|
||||
namespace ServerMonitorManager.Control;
|
||||
|
||||
public sealed class ControlStore(IOptions<ControlOptions> options)
|
||||
{
|
||||
private readonly string _connectionString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = options.Value.DatabasePath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Cache = SqliteCacheMode.Shared
|
||||
}.ToString();
|
||||
|
||||
public async Task InitializeAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var path = new SqliteConnectionStringBuilder(_connectionString).DataSource;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
CREATE TABLE IF NOT EXISTS enrollment_tokens (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
consumed_at TEXT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
certificate_thumbprint TEXT NOT NULL UNIQUE,
|
||||
certificate_expires_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
agent_version TEXT NOT NULL DEFAULT '',
|
||||
last_seen_at TEXT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS metric_samples (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id TEXT NOT NULL REFERENCES agents(node_id) ON DELETE CASCADE,
|
||||
recorded_at TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_metric_samples_node_time
|
||||
ON metric_samples(node_id, recorded_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS idempotency (
|
||||
operation_key TEXT PRIMARY KEY,
|
||||
request_hash TEXT NOT NULL,
|
||||
response_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
recorded_at TEXT NOT NULL,
|
||||
actor TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
subject TEXT NOT NULL,
|
||||
details_json TEXT NOT NULL
|
||||
);
|
||||
""";
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<string> CreateEnrollmentTokenAsync(
|
||||
string nodeId,
|
||||
TimeSpan lifetime,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var token = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32))
|
||||
.TrimEnd('=').Replace('+', '-').Replace('/', '_');
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
INSERT INTO enrollment_tokens(token_hash, node_id, expires_at)
|
||||
VALUES ($hash, $node, $expires);
|
||||
""";
|
||||
command.Parameters.AddWithValue("$hash", Hash(token));
|
||||
command.Parameters.AddWithValue("$node", nodeId);
|
||||
command.Parameters.AddWithValue("$expires", DateTimeOffset.UtcNow.Add(lifetime).ToString("O"));
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
return token;
|
||||
}
|
||||
|
||||
public async Task<EnrollmentResponse?> EnrollAsync(
|
||||
EnrollmentRequest request,
|
||||
Func<IssuedCertificate> issueCertificate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||
|
||||
var cached = await ReadIdempotentAsync<EnrollmentResponse>(
|
||||
connection,
|
||||
transaction,
|
||||
$"enroll:{request.IdempotencyKey}",
|
||||
Fingerprint(request, SmmJsonContext.Default.EnrollmentRequest),
|
||||
SmmJsonContext.Default.EnrollmentResponse,
|
||||
cancellationToken);
|
||||
if (cached is not null)
|
||||
{
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return cached;
|
||||
}
|
||||
|
||||
var consume = connection.CreateCommand();
|
||||
consume.Transaction = transaction;
|
||||
consume.CommandText = """
|
||||
UPDATE enrollment_tokens
|
||||
SET consumed_at = $now
|
||||
WHERE token_hash = $hash
|
||||
AND node_id = $node
|
||||
AND consumed_at IS NULL
|
||||
AND expires_at >= $now;
|
||||
""";
|
||||
var now = DateTimeOffset.UtcNow.ToString("O");
|
||||
consume.Parameters.AddWithValue("$now", now);
|
||||
consume.Parameters.AddWithValue("$hash", Hash(request.Token));
|
||||
consume.Parameters.AddWithValue("$node", request.NodeId);
|
||||
if (await consume.ExecuteNonQueryAsync(cancellationToken) != 1)
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
return null;
|
||||
}
|
||||
|
||||
var issued = issueCertificate();
|
||||
var response = new EnrollmentResponse(
|
||||
request.NodeId,
|
||||
issued.CertificatePem,
|
||||
issued.CertificateAuthorityPem,
|
||||
issued.ExpiresAt);
|
||||
var upsert = connection.CreateCommand();
|
||||
upsert.Transaction = transaction;
|
||||
upsert.CommandText = """
|
||||
INSERT INTO agents(node_id, name, certificate_thumbprint, certificate_expires_at, status)
|
||||
VALUES ($node, $node, $thumbprint, $expires, 'Enrolled')
|
||||
ON CONFLICT(node_id) DO UPDATE SET
|
||||
certificate_thumbprint = excluded.certificate_thumbprint,
|
||||
certificate_expires_at = excluded.certificate_expires_at,
|
||||
status = 'Enrolled';
|
||||
""";
|
||||
upsert.Parameters.AddWithValue("$node", request.NodeId);
|
||||
upsert.Parameters.AddWithValue("$thumbprint", issued.Thumbprint);
|
||||
upsert.Parameters.AddWithValue("$expires", issued.ExpiresAt.ToString("O"));
|
||||
await upsert.ExecuteNonQueryAsync(cancellationToken);
|
||||
await WriteIdempotentAsync(
|
||||
connection,
|
||||
transaction,
|
||||
$"enroll:{request.IdempotencyKey}",
|
||||
Fingerprint(request, SmmJsonContext.Default.EnrollmentRequest),
|
||||
response,
|
||||
SmmJsonContext.Default.EnrollmentResponse,
|
||||
cancellationToken);
|
||||
await WriteAuditAsync(connection, transaction, request.NodeId, "agent.enroll", request.NodeId, "{}", cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return response;
|
||||
}
|
||||
|
||||
public async Task<bool> IsCertificateActiveAsync(
|
||||
string thumbprint,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM agents
|
||||
WHERE certificate_thumbprint = $thumbprint
|
||||
AND certificate_expires_at > $now
|
||||
AND status != 'Revoked');
|
||||
""";
|
||||
command.Parameters.AddWithValue("$thumbprint", thumbprint);
|
||||
command.Parameters.AddWithValue("$now", DateTimeOffset.UtcNow.ToString("O"));
|
||||
return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)) == 1;
|
||||
}
|
||||
|
||||
public async Task<bool> IsCertificateForNodeAsync(
|
||||
string thumbprint,
|
||||
string nodeId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM agents
|
||||
WHERE certificate_thumbprint = $thumbprint
|
||||
AND node_id = $node
|
||||
AND certificate_expires_at > $now
|
||||
AND status != 'Revoked');
|
||||
""";
|
||||
command.Parameters.AddWithValue("$thumbprint", thumbprint);
|
||||
command.Parameters.AddWithValue("$node", nodeId);
|
||||
command.Parameters.AddWithValue("$now", DateTimeOffset.UtcNow.ToString("O"));
|
||||
return Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)) == 1;
|
||||
}
|
||||
|
||||
public async Task<AgentHeartbeatResponse> RecordHeartbeatAsync(
|
||||
AgentHeartbeat heartbeat,
|
||||
int nextHeartbeatSeconds,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken);
|
||||
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken);
|
||||
var cached = await ReadIdempotentAsync<AgentHeartbeatResponse>(
|
||||
connection,
|
||||
transaction,
|
||||
$"heartbeat:{heartbeat.NodeId}:{heartbeat.IdempotencyKey}",
|
||||
Fingerprint(heartbeat, SmmJsonContext.Default.AgentHeartbeat),
|
||||
SmmJsonContext.Default.AgentHeartbeatResponse,
|
||||
cancellationToken);
|
||||
if (cached is not null)
|
||||
{
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return cached;
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var update = connection.CreateCommand();
|
||||
update.Transaction = transaction;
|
||||
update.CommandText = """
|
||||
UPDATE agents
|
||||
SET status = 'Online', agent_version = $version, last_seen_at = $now
|
||||
WHERE node_id = $node;
|
||||
""";
|
||||
update.Parameters.AddWithValue("$version", heartbeat.AgentVersion);
|
||||
update.Parameters.AddWithValue("$now", now.ToString("O"));
|
||||
update.Parameters.AddWithValue("$node", heartbeat.NodeId);
|
||||
if (await update.ExecuteNonQueryAsync(cancellationToken) != 1)
|
||||
{
|
||||
throw new InvalidOperationException("Unknown agent node id.");
|
||||
}
|
||||
|
||||
var insert = connection.CreateCommand();
|
||||
insert.Transaction = transaction;
|
||||
insert.CommandText = """
|
||||
INSERT INTO metric_samples(node_id, recorded_at, payload_json)
|
||||
VALUES ($node, $now, $payload);
|
||||
SELECT last_insert_rowid();
|
||||
""";
|
||||
insert.Parameters.AddWithValue("$node", heartbeat.NodeId);
|
||||
insert.Parameters.AddWithValue("$now", now.ToString("O"));
|
||||
insert.Parameters.AddWithValue(
|
||||
"$payload",
|
||||
JsonSerializer.Serialize(heartbeat, SmmJsonContext.Default.AgentHeartbeat));
|
||||
var sequence = Convert.ToInt64(await insert.ExecuteScalarAsync(cancellationToken));
|
||||
var response = new AgentHeartbeatResponse(now, sequence, nextHeartbeatSeconds);
|
||||
await WriteIdempotentAsync(
|
||||
connection,
|
||||
transaction,
|
||||
$"heartbeat:{heartbeat.NodeId}:{heartbeat.IdempotencyKey}",
|
||||
Fingerprint(heartbeat, SmmJsonContext.Default.AgentHeartbeat),
|
||||
response,
|
||||
SmmJsonContext.Default.AgentHeartbeatResponse,
|
||||
cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return response;
|
||||
}
|
||||
|
||||
private async Task<SqliteConnection> OpenAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = new SqliteConnection(_connectionString);
|
||||
await connection.OpenAsync(cancellationToken);
|
||||
return connection;
|
||||
}
|
||||
|
||||
private static string Hash(string value)
|
||||
=> Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value)));
|
||||
|
||||
private static string Fingerprint<T>(T value, JsonTypeInfo<T> typeInfo)
|
||||
=> Hash(JsonSerializer.Serialize(value, typeInfo));
|
||||
|
||||
private static async Task<T?> ReadIdempotentAsync<T>(
|
||||
SqliteConnection connection,
|
||||
SqliteTransaction transaction,
|
||||
string key,
|
||||
string requestHash,
|
||||
JsonTypeInfo<T> typeInfo,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var command = connection.CreateCommand();
|
||||
command.Transaction = transaction;
|
||||
command.CommandText = "SELECT request_hash, response_json FROM idempotency WHERE operation_key = $key;";
|
||||
command.Parameters.AddWithValue("$key", key);
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
if (!await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
return default;
|
||||
}
|
||||
if (!string.Equals(reader.GetString(0), requestHash, StringComparison.Ordinal))
|
||||
{
|
||||
throw new IdempotencyConflictException();
|
||||
}
|
||||
return JsonSerializer.Deserialize(reader.GetString(1), typeInfo);
|
||||
}
|
||||
|
||||
private static async Task WriteIdempotentAsync<T>(
|
||||
SqliteConnection connection,
|
||||
SqliteTransaction transaction,
|
||||
string key,
|
||||
string requestHash,
|
||||
T response,
|
||||
JsonTypeInfo<T> typeInfo,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var command = connection.CreateCommand();
|
||||
command.Transaction = transaction;
|
||||
command.CommandText = """
|
||||
INSERT INTO idempotency(operation_key, request_hash, response_json, created_at)
|
||||
VALUES ($key, $requestHash, $response, $now);
|
||||
""";
|
||||
command.Parameters.AddWithValue("$key", key);
|
||||
command.Parameters.AddWithValue("$requestHash", requestHash);
|
||||
command.Parameters.AddWithValue("$response", JsonSerializer.Serialize(response, typeInfo));
|
||||
command.Parameters.AddWithValue("$now", DateTimeOffset.UtcNow.ToString("O"));
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task WriteAuditAsync(
|
||||
SqliteConnection connection,
|
||||
SqliteTransaction transaction,
|
||||
string actor,
|
||||
string action,
|
||||
string subject,
|
||||
string details,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var command = connection.CreateCommand();
|
||||
command.Transaction = transaction;
|
||||
command.CommandText = """
|
||||
INSERT INTO audit(recorded_at, actor, action, subject, details_json)
|
||||
VALUES ($now, $actor, $action, $subject, $details);
|
||||
""";
|
||||
command.Parameters.AddWithValue("$now", DateTimeOffset.UtcNow.ToString("O"));
|
||||
command.Parameters.AddWithValue("$actor", actor);
|
||||
command.Parameters.AddWithValue("$action", action);
|
||||
command.Parameters.AddWithValue("$subject", subject);
|
||||
command.Parameters.AddWithValue("$details", details);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class IdempotencyConflictException : Exception
|
||||
{
|
||||
public IdempotencyConflictException()
|
||||
: base("Idempotency key was reused with a different request.")
|
||||
{
|
||||
}
|
||||
}
|
||||
217
src/ServerMonitorManager.Control/Program.cs
Normal file
217
src/ServerMonitorManager.Control/Program.cs
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
using System.Security.Claims;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Threading.RateLimiting;
|
||||
using Microsoft.AspNetCore.Authentication.Certificate;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Https;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ServerMonitorManager.Control;
|
||||
using ServerMonitorManager.Core;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddProblemDetails();
|
||||
builder.Services.AddHealthChecks();
|
||||
builder.Services.AddRateLimiter(options =>
|
||||
{
|
||||
options.RejectionStatusCode = StatusCodes.Status429TooManyRequests;
|
||||
options.AddPolicy("enrollment", context => RateLimitPartition.GetFixedWindowLimiter(
|
||||
context.Connection.RemoteIpAddress?.ToString() ?? "unknown",
|
||||
_ => new FixedWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 10,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
QueueLimit = 0,
|
||||
AutoReplenishment = true
|
||||
}));
|
||||
});
|
||||
builder.Services.ConfigureHttpJsonOptions(options =>
|
||||
options.SerializerOptions.TypeInfoResolverChain.Insert(0, SmmJsonContext.Default));
|
||||
builder.Services.AddOptions<ControlOptions>()
|
||||
.Bind(builder.Configuration.GetSection(ControlOptions.SectionName))
|
||||
.Validate(options =>
|
||||
!string.IsNullOrWhiteSpace(options.DatabasePath)
|
||||
&& !string.IsNullOrWhiteSpace(options.CertificateAuthorityPath)
|
||||
&& options.HeartbeatSeconds is >= 10 and <= 300,
|
||||
"Control paths are required and HeartbeatSeconds must be between 10 and 300.")
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton<ControlStore>();
|
||||
builder.Services.AddSingleton<CertificateAuthority>();
|
||||
builder.Services.AddAuthentication(CertificateAuthenticationDefaults.AuthenticationScheme)
|
||||
.AddCertificate(options =>
|
||||
{
|
||||
options.AllowedCertificateTypes = CertificateTypes.All;
|
||||
options.RevocationMode = X509RevocationMode.NoCheck;
|
||||
options.ValidateCertificateUse = true;
|
||||
options.ValidateValidityPeriod = true;
|
||||
options.Events = new CertificateAuthenticationEvents
|
||||
{
|
||||
OnCertificateValidated = async context =>
|
||||
{
|
||||
var store = context.HttpContext.RequestServices.GetRequiredService<ControlStore>();
|
||||
if (!await store.IsCertificateActiveAsync(context.ClientCertificate.Thumbprint))
|
||||
{
|
||||
context.Fail("The agent certificate is unknown, expired, or revoked.");
|
||||
return;
|
||||
}
|
||||
|
||||
context.Principal = new ClaimsPrincipal(new ClaimsIdentity(
|
||||
[new Claim(ClaimTypes.NameIdentifier, context.ClientCertificate.Thumbprint)],
|
||||
context.Scheme.Name));
|
||||
context.Success();
|
||||
}
|
||||
};
|
||||
});
|
||||
builder.Services.AddOptions<CertificateAuthenticationOptions>(
|
||||
CertificateAuthenticationDefaults.AuthenticationScheme)
|
||||
.Configure<CertificateAuthority>((options, authority) =>
|
||||
{
|
||||
options.ChainTrustValidationMode = X509ChainTrustMode.CustomRootTrust;
|
||||
options.CustomTrustStore.Add(authority.PublicCertificate);
|
||||
});
|
||||
builder.Services.AddAuthorization();
|
||||
builder.WebHost.ConfigureKestrel(options =>
|
||||
{
|
||||
options.Limits.MaxRequestBodySize = 64 * 1024;
|
||||
options.ConfigureHttpsDefaults(https =>
|
||||
https.ClientCertificateMode = ClientCertificateMode.AllowCertificate);
|
||||
});
|
||||
|
||||
var app = builder.Build();
|
||||
app.UseExceptionHandler();
|
||||
app.UseHsts();
|
||||
app.UseRateLimiter();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
|
||||
var store = app.Services.GetRequiredService<ControlStore>();
|
||||
await store.InitializeAsync();
|
||||
|
||||
if (args is ["token-create", var nodeId])
|
||||
{
|
||||
if (!NodeIdValidator.IsValid(nodeId))
|
||||
{
|
||||
Console.Error.WriteLine("Node id must contain 1-63 lowercase letters, digits, or hyphens.");
|
||||
return 2;
|
||||
}
|
||||
|
||||
Console.WriteLine(await store.CreateEnrollmentTokenAsync(nodeId, TimeSpan.FromMinutes(10)));
|
||||
return 0;
|
||||
}
|
||||
|
||||
app.MapHealthChecks("/healthz").AllowAnonymous();
|
||||
|
||||
app.MapPost("/api/v1/enroll", async (
|
||||
EnrollmentRequest request,
|
||||
ControlStore controlStore,
|
||||
CertificateAuthority authority,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!NodeIdValidator.IsValid(request.NodeId)
|
||||
|| string.IsNullOrWhiteSpace(request.Token)
|
||||
|| string.IsNullOrWhiteSpace(request.CertificateSigningRequestPem)
|
||||
|| !IdempotencyKeyValidator.IsValid(request.IdempotencyKey))
|
||||
{
|
||||
return Results.ValidationProblem(new Dictionary<string, string[]>
|
||||
{
|
||||
["request"] = ["Invalid enrollment request."]
|
||||
});
|
||||
}
|
||||
|
||||
EnrollmentResponse? response;
|
||||
try
|
||||
{
|
||||
response = await controlStore.EnrollAsync(
|
||||
request,
|
||||
() => authority.IssueClientCertificate(request.NodeId, request.CertificateSigningRequestPem),
|
||||
cancellationToken);
|
||||
}
|
||||
catch (IdempotencyConflictException)
|
||||
{
|
||||
return Results.Conflict(new ProblemDetails
|
||||
{
|
||||
Title = "Idempotency key conflict",
|
||||
Status = StatusCodes.Status409Conflict
|
||||
});
|
||||
}
|
||||
catch (Exception exception) when (exception is CryptographicException or InvalidOperationException)
|
||||
{
|
||||
return Results.ValidationProblem(new Dictionary<string, string[]>
|
||||
{
|
||||
["certificateSigningRequestPem"] = ["Invalid certificate signing request."]
|
||||
});
|
||||
}
|
||||
|
||||
return response is null
|
||||
? Results.Unauthorized()
|
||||
: Results.Ok(response);
|
||||
}).AllowAnonymous().RequireRateLimiting("enrollment");
|
||||
|
||||
var agents = app.MapGroup("/api/v1/agents").RequireAuthorization();
|
||||
agents.MapPost("/heartbeat", async (
|
||||
AgentHeartbeat heartbeat,
|
||||
HttpContext context,
|
||||
ControlStore controlStore,
|
||||
IOptions<ControlOptions> options,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!NodeIdValidator.IsValid(heartbeat.NodeId)
|
||||
|| !IdempotencyKeyValidator.IsValid(heartbeat.IdempotencyKey)
|
||||
|| heartbeat.SentAt < DateTimeOffset.UtcNow.AddMinutes(-5)
|
||||
|| heartbeat.SentAt > DateTimeOffset.UtcNow.AddMinutes(1))
|
||||
{
|
||||
return Results.ValidationProblem(new Dictionary<string, string[]>
|
||||
{
|
||||
["heartbeat"] = ["Invalid or stale heartbeat."]
|
||||
});
|
||||
}
|
||||
|
||||
var certificate = await context.Connection.GetClientCertificateAsync(cancellationToken);
|
||||
if (certificate is null)
|
||||
{
|
||||
return Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!await controlStore.IsCertificateForNodeAsync(
|
||||
certificate.Thumbprint,
|
||||
heartbeat.NodeId,
|
||||
cancellationToken))
|
||||
{
|
||||
return Results.Forbid();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var response = await controlStore.RecordHeartbeatAsync(
|
||||
heartbeat,
|
||||
options.Value.HeartbeatSeconds,
|
||||
cancellationToken);
|
||||
return Results.Ok(response);
|
||||
}
|
||||
catch (IdempotencyConflictException)
|
||||
{
|
||||
return Results.Conflict(new ProblemDetails
|
||||
{
|
||||
Title = "Idempotency key conflict",
|
||||
Status = StatusCodes.Status409Conflict
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await app.RunAsync();
|
||||
return 0;
|
||||
|
||||
public partial class Program;
|
||||
|
||||
internal static class NodeIdValidator
|
||||
{
|
||||
public static bool IsValid(string value)
|
||||
=> value.Length is >= 1 and <= 63
|
||||
&& value.All(character => character is >= 'a' and <= 'z' or >= '0' and <= '9' or '-');
|
||||
}
|
||||
|
||||
internal static class IdempotencyKeyValidator
|
||||
{
|
||||
public static bool IsValid(string value)
|
||||
=> Guid.TryParse(value, out _);
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
<Version>0.1.0</Version>
|
||||
<AssemblyName>ochenstarik-smm-control</AssemblyName>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ServerMonitorManager.Core\ServerMonitorManager.Core.csproj" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Certificate" Version="10.0.10" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.10" />
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.12" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
15
src/ServerMonitorManager.Control/appsettings.json
Normal file
15
src/ServerMonitorManager.Control/appsettings.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"Control": {
|
||||
"DatabasePath": "/var/lib/ochenstarik-server-monitor-manager/control.db",
|
||||
"CertificateAuthorityPath": "/etc/ochenstarik-server-monitor-manager/control-ca.pfx",
|
||||
"CertificateAuthorityPassword": null,
|
||||
"HeartbeatSeconds": 30
|
||||
},
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning"
|
||||
}
|
||||
},
|
||||
"AllowedHosts": "*"
|
||||
}
|
||||
39
src/ServerMonitorManager.Core/Contracts.cs
Normal file
39
src/ServerMonitorManager.Core/Contracts.cs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
namespace ServerMonitorManager.Core;
|
||||
|
||||
public sealed record EnrollmentRequest(
|
||||
string NodeId,
|
||||
string Token,
|
||||
string CertificateSigningRequestPem,
|
||||
string IdempotencyKey);
|
||||
|
||||
public sealed record EnrollmentResponse(
|
||||
string NodeId,
|
||||
string CertificatePem,
|
||||
string CertificateAuthorityPem,
|
||||
DateTimeOffset ExpiresAt);
|
||||
|
||||
public sealed record AgentHeartbeat(
|
||||
string NodeId,
|
||||
string AgentVersion,
|
||||
DateTimeOffset SentAt,
|
||||
double LoadOne,
|
||||
long MemoryUsedBytes,
|
||||
long MemoryTotalBytes,
|
||||
long DiskUsedBytes,
|
||||
long DiskTotalBytes,
|
||||
long NetworkReceiveBytes,
|
||||
long NetworkTransmitBytes,
|
||||
long UptimeSeconds,
|
||||
string IdempotencyKey);
|
||||
|
||||
public sealed record AgentHeartbeatResponse(
|
||||
DateTimeOffset AcceptedAt,
|
||||
long Sequence,
|
||||
int NextHeartbeatSeconds);
|
||||
|
||||
public sealed record AgentSummary(
|
||||
string NodeId,
|
||||
string Name,
|
||||
string Status,
|
||||
string AgentVersion,
|
||||
DateTimeOffset? LastSeenAt);
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
10
src/ServerMonitorManager.Core/SmmJsonContext.cs
Normal file
10
src/ServerMonitorManager.Core/SmmJsonContext.cs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace ServerMonitorManager.Core;
|
||||
|
||||
[JsonSerializable(typeof(EnrollmentRequest))]
|
||||
[JsonSerializable(typeof(EnrollmentResponse))]
|
||||
[JsonSerializable(typeof(AgentHeartbeat))]
|
||||
[JsonSerializable(typeof(AgentHeartbeatResponse))]
|
||||
[JsonSerializable(typeof(AgentSummary[]))]
|
||||
public sealed partial class SmmJsonContext : JsonSerializerContext;
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ServerMonitorManager.Control;
|
||||
using Xunit;
|
||||
|
||||
namespace ServerMonitorManager.Control.Tests;
|
||||
|
||||
public sealed class CertificateAuthorityTests : IDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-ca-tests-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public void IssuedAgentCertificateChainsToControlAuthority()
|
||||
{
|
||||
Directory.CreateDirectory(_directory);
|
||||
var caPath = Path.Combine(_directory, "control-ca.pfx");
|
||||
using var caKey = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var caRequest = new CertificateRequest("CN=SMM Test CA", caKey, HashAlgorithmName.SHA256);
|
||||
caRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true));
|
||||
caRequest.CertificateExtensions.Add(new X509KeyUsageExtension(
|
||||
X509KeyUsageFlags.KeyCertSign | X509KeyUsageFlags.CrlSign,
|
||||
true));
|
||||
using var ca = caRequest.CreateSelfSigned(
|
||||
DateTimeOffset.UtcNow.AddMinutes(-1),
|
||||
DateTimeOffset.UtcNow.AddYears(2));
|
||||
File.WriteAllBytes(caPath, ca.Export(X509ContentType.Pfx));
|
||||
|
||||
using var authority = new CertificateAuthority(Options.Create(new ControlOptions
|
||||
{
|
||||
DatabasePath = Path.Combine(_directory, "unused.db"),
|
||||
CertificateAuthorityPath = caPath
|
||||
}));
|
||||
using var agentKey = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var agentRequest = new CertificateRequest("CN=home", agentKey, HashAlgorithmName.SHA256);
|
||||
var issued = authority.IssueClientCertificate("home", agentRequest.CreateSigningRequestPem());
|
||||
using var agentCertificate = X509Certificate2.CreateFromPem(
|
||||
issued.CertificatePem,
|
||||
agentKey.ExportPkcs8PrivateKeyPem());
|
||||
using var chain = new X509Chain();
|
||||
chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
|
||||
chain.ChainPolicy.CustomTrustStore.Add(authority.PublicCertificate);
|
||||
chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
|
||||
|
||||
Assert.True(chain.Build(agentCertificate));
|
||||
Assert.Contains(
|
||||
agentCertificate.Extensions.OfType<X509EnhancedKeyUsageExtension>()
|
||||
.SelectMany(extension => extension.EnhancedKeyUsages.Cast<Oid>()),
|
||||
oid => oid.Value == "1.3.6.1.5.5.7.3.2");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_directory))
|
||||
{
|
||||
Directory.Delete(_directory, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using ServerMonitorManager.Control;
|
||||
using ServerMonitorManager.Core;
|
||||
using Xunit;
|
||||
|
||||
namespace ServerMonitorManager.Control.Tests;
|
||||
|
||||
public sealed class ControlStoreTests : IAsyncDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), $"smm-tests-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public async Task EnrollmentTokenIsAtomicAndIdempotent()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
var token = await store.CreateEnrollmentTokenAsync("home", TimeSpan.FromMinutes(10), cancellationToken);
|
||||
var idempotencyKey = Guid.NewGuid().ToString();
|
||||
var request = new EnrollmentRequest("home", token, "unused-in-store-test", idempotencyKey);
|
||||
var issued = new IssuedCertificate("certificate", "ca", "AA11", DateTimeOffset.UtcNow.AddYears(1));
|
||||
|
||||
var first = await store.EnrollAsync(request, () => issued, cancellationToken);
|
||||
var retry = await store.EnrollAsync(request, () => throw new InvalidOperationException("must use cache"), cancellationToken);
|
||||
var replay = await store.EnrollAsync(
|
||||
request with { IdempotencyKey = Guid.NewGuid().ToString() },
|
||||
() => issued,
|
||||
cancellationToken);
|
||||
|
||||
Assert.NotNull(first);
|
||||
Assert.Equal(first, retry);
|
||||
Assert.Null(replay);
|
||||
Assert.True(await store.IsCertificateForNodeAsync("AA11", "home", cancellationToken));
|
||||
Assert.False(await store.IsCertificateForNodeAsync("AA11", "other", cancellationToken));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HeartbeatRetryDoesNotDuplicateMetricSample()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var store = CreateStore();
|
||||
await store.InitializeAsync(cancellationToken);
|
||||
var token = await store.CreateEnrollmentTokenAsync("home", TimeSpan.FromMinutes(10), cancellationToken);
|
||||
var issued = new IssuedCertificate("certificate", "ca", "BB22", DateTimeOffset.UtcNow.AddYears(1));
|
||||
await store.EnrollAsync(
|
||||
new EnrollmentRequest("home", token, "csr", Guid.NewGuid().ToString()),
|
||||
() => issued,
|
||||
cancellationToken);
|
||||
var heartbeat = new AgentHeartbeat(
|
||||
"home", "test", DateTimeOffset.UtcNow, 0.5, 1, 2, 3, 4, 5, 6, 7,
|
||||
Guid.NewGuid().ToString());
|
||||
|
||||
var first = await store.RecordHeartbeatAsync(heartbeat, 30, cancellationToken);
|
||||
var retry = await store.RecordHeartbeatAsync(heartbeat, 30, cancellationToken);
|
||||
|
||||
Assert.Equal(first, retry);
|
||||
Assert.Equal(1, first.Sequence);
|
||||
await Assert.ThrowsAsync<IdempotencyConflictException>(() => store.RecordHeartbeatAsync(
|
||||
heartbeat with { LoadOne = 0.9 },
|
||||
30,
|
||||
cancellationToken));
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
if (Directory.Exists(_directory))
|
||||
{
|
||||
Directory.Delete(_directory, recursive: true);
|
||||
}
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private ControlStore CreateStore()
|
||||
{
|
||||
Directory.CreateDirectory(_directory);
|
||||
return new ControlStore(Options.Create(new ControlOptions
|
||||
{
|
||||
DatabasePath = Path.Combine(_directory, "control.db"),
|
||||
CertificateAuthorityPath = Path.Combine(_directory, "unused.pfx")
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\ServerMonitorManager.Control\ServerMonitorManager.Control.csproj" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
|
||||
<PackageReference Include="xunit.v3" Version="3.2.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Loading…
Reference in a new issue