feat: Supervisor Agent — мониторинг и авто-переключение профилей
This commit is contained in:
ochenstarik-ui
parent
3f9a925f24
commit
f97afdc399
2 changed files with 459 additions and 0 deletions
179
docs/adapters/SUPERVISOR.md
Normal file
179
docs/adapters/SUPERVISOR.md
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
# Supervisor Agent — надзиратель провайдеров
|
||||
|
||||
**Версия:** 0.1.0-draft | **Статус:** Draft
|
||||
|
||||
## Назначение
|
||||
|
||||
Supervisor Agent — специализированный агент-надзиратель, который:
|
||||
1. Мониторит здоровье всех провайдеров/моделей
|
||||
2. Обнаруживает исчерпание квот (429, 402, 403)
|
||||
3. Переключает worker-профили на fallback-модели
|
||||
4. Возвращает профили на основные модели после восстановления квот
|
||||
5. Ведёт лог всех инцидентов и переключений
|
||||
|
||||
## Архитектура
|
||||
|
||||
```
|
||||
Supervisor Agent (cron: каждые 5 мин)
|
||||
│
|
||||
├─ check_provider(provider) → status
|
||||
│ ├─ HTTP health probe
|
||||
│ ├─ quota check (usage API)
|
||||
│ └─ latency check
|
||||
│
|
||||
├─ detect_incident(status) → action
|
||||
│ ├─ 429 → switch to fallback
|
||||
│ ├─ 402 → switch to fallback
|
||||
│ ├─ timeout → mark degraded
|
||||
│ └─ recovery → switch back to primary
|
||||
│
|
||||
├─ apply_action(profile, new_model)
|
||||
│ └─ hermes config set model.default <new_model> -p <profile>
|
||||
│
|
||||
└─ log_incident(incident)
|
||||
└─ supervisor.log + Telegram notification
|
||||
```
|
||||
|
||||
## Модель инцидента
|
||||
|
||||
```yaml
|
||||
incident:
|
||||
id: uuid
|
||||
timestamp: ISO8601
|
||||
provider: opencode-go|grok-cli|gemini|nvidia|ollama
|
||||
model: kimi-k2.7-code
|
||||
error_code: 429|402|403|timeout
|
||||
action: switch_to_fallback|switch_back|alert_only
|
||||
profile_affected: worker-code|worker-fast|worker-research|worker-review
|
||||
old_model: opencode-go/kimi-k2.7-code
|
||||
new_model: grok-cli/grok-4.5
|
||||
recovery_eta: "2026-07-21T00:00:00Z"
|
||||
status: active|resolved
|
||||
```
|
||||
|
||||
## Матрица переключений
|
||||
|
||||
| Профиль | Primary | Fallback 1 | Fallback 2 | Last Resort |
|
||||
|---------|---------|------------|------------|-------------|
|
||||
| worker-code | opencode-go/kimi | grok-4.5 | gemini-3-flash | opencode-go (любая) |
|
||||
| worker-fast | grok-4.5 | opencode-go/kimi | gemini-3-flash | opencode-go |
|
||||
| worker-research | gemini-3-flash | opencode-go/kimi | grok-4.5 | opencode-go |
|
||||
| worker-review | nvidia/deepseek-v4 | opencode-go/kimi | gemini-3-flash | opencode-go |
|
||||
|
||||
## Конфигурация supervisor
|
||||
|
||||
```yaml
|
||||
# supervisor.yaml (рядом с Hermes config)
|
||||
supervisor:
|
||||
enabled: true
|
||||
check_interval: 300 # секунд (5 минут)
|
||||
health_timeout: 15 # секунд на health probe
|
||||
cooldown_minutes: 30 # не переключать чаще чем раз в 30 мин
|
||||
notify:
|
||||
telegram: true
|
||||
log: true
|
||||
providers:
|
||||
opencode-go:
|
||||
health_url: http://localhost:20127/v1/models
|
||||
quota_reset: "daily" # когда сбрасывается квота
|
||||
grok-cli:
|
||||
health_url: http://localhost:20127/v1/models
|
||||
quota_reset: "monthly"
|
||||
gemini:
|
||||
health_url: http://localhost:20127/v1/models
|
||||
quota_reset: "daily"
|
||||
nvidia:
|
||||
health_url: http://localhost:20127/v1/models
|
||||
quota_reset: "daily"
|
||||
fallback_matrix:
|
||||
worker-code:
|
||||
primary: opencode-go/kimi-k2.7-code
|
||||
fallbacks:
|
||||
- grok-cli/grok-4.5
|
||||
- gemini/gemini-3-flash-preview
|
||||
worker-fast:
|
||||
primary: grok-cli/grok-4.5
|
||||
fallbacks:
|
||||
- opencode-go/kimi-k2.7-code
|
||||
- gemini/gemini-3-flash-preview
|
||||
worker-research:
|
||||
primary: gemini/gemini-3-flash-preview
|
||||
fallbacks:
|
||||
- opencode-go/kimi-k2.7-code
|
||||
worker-review:
|
||||
primary: nvidia/deepseek-ai/deepseek-v4-pro
|
||||
fallbacks:
|
||||
- opencode-go/kimi-k2.7-code
|
||||
```
|
||||
|
||||
## Алгоритм работы
|
||||
|
||||
```
|
||||
1. КАЖДЫЕ check_interval секунд:
|
||||
a. Для каждого провайдера: POST health_url → статус
|
||||
b. Если 429/402/403 → INCIDENT
|
||||
c. Если timeout × 3 → DEGRADED
|
||||
|
||||
2. ПРИ ИНЦИДЕНТЕ:
|
||||
a. Проверить cooldown — если последнее переключение < 30 мин назад → alert_only
|
||||
b. Найти все профили с этим провайдером как primary
|
||||
c. Для каждого: переключить на первый доступный fallback
|
||||
d. Записать incident в лог
|
||||
e. Уведомить в Telegram
|
||||
|
||||
3. ПРИ ВОССТАНОВЛЕНИИ:
|
||||
a. Проверить primary провайдер 3 раза с интервалом 60с
|
||||
b. Если все 3 проверки OK → вернуть профили на primary
|
||||
c. Обновить incident.status = resolved
|
||||
```
|
||||
|
||||
## Реализация
|
||||
|
||||
### Как cron-задача Hermes
|
||||
|
||||
```bash
|
||||
hermes cron create "*/5 * * * *" \
|
||||
--name "supervisor-health-check" \
|
||||
--script scripts/supervisor.py \
|
||||
--no_agent # script-only, не тратит токены LLM
|
||||
```
|
||||
|
||||
### supervisor.py (псевдокод)
|
||||
|
||||
```python
|
||||
def check_provider(health_url, api_key):
|
||||
r = requests.post(health_url, headers={"Authorization": f"Bearer {api_key}"},
|
||||
json={"model": "test", "messages": [{"role":"user","content":"Hi"}], "max_tokens":1})
|
||||
if r.status_code == 429:
|
||||
return "quota_exceeded"
|
||||
if r.status_code == 402:
|
||||
return "payment_required"
|
||||
if r.status_code != 200:
|
||||
return "error"
|
||||
return "ok"
|
||||
|
||||
def switch_profile(profile, model):
|
||||
subprocess.run(["hermes", "config", "set", "model.default", model, "-p", profile])
|
||||
|
||||
def main():
|
||||
for provider in config["providers"]:
|
||||
status = check_provider(provider["health_url"], API_KEY)
|
||||
if status != "ok":
|
||||
# Найти затронутые профили и переключить
|
||||
for profile, fb in config["fallback_matrix"].items():
|
||||
if fb["primary"].startswith(provider):
|
||||
for fallback_model in fb["fallbacks"]:
|
||||
if check_provider(...) == "ok":
|
||||
switch_profile(profile, fallback_model)
|
||||
notify_telegram(f"{profile}: {fb['primary']} → {fallback_model}")
|
||||
break
|
||||
```
|
||||
|
||||
## Интеграция с Agent Control Center
|
||||
|
||||
Supervisor Agent — это Connector типа `hermes-cron` с адаптером `supervisor`.
|
||||
В терминах ACC:
|
||||
- **Agent:** supervisor (runtime: hermes-cron, model: none — script-only)
|
||||
- **Capabilities:** provider_health_check, profile_switch, incident_logging
|
||||
- **Connector:** локальный (встроен в Hermes)
|
||||
- **Approvals:** не требуются (переключение модели — низкий риск)
|
||||
280
scripts/supervisor.py
Normal file
280
scripts/supervisor.py
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Supervisor Agent — мониторинг и авто-переключение worker-профилей при отказе провайдеров.
|
||||
|
||||
Запуск:
|
||||
python scripts/supervisor.py
|
||||
или как cron: hermes cron create "*/5 * * * *" --script scripts/supervisor.py --no_agent
|
||||
|
||||
Конфигурация: supervisor.yaml (рядом с Hermes config)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
HERMES_HOME = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes"))
|
||||
CONFIG_PATH = HERMES_HOME / "supervisor.yaml"
|
||||
INCIDENT_LOG = HERMES_HOME / "logs" / "supervisor.log"
|
||||
STATE_FILE = HERMES_HOME / "cache" / "supervisor_state.json"
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────
|
||||
|
||||
def log(msg: str) -> None:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"[{timestamp}] {msg}"
|
||||
print(line, file=sys.stderr)
|
||||
INCIDENT_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(INCIDENT_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def load_state() -> dict:
|
||||
if STATE_FILE.exists():
|
||||
return json.loads(STATE_FILE.read_text())
|
||||
return {"incidents": [], "last_switch": {}, "last_check": {}}
|
||||
|
||||
def save_state(state: dict) -> None:
|
||||
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
STATE_FILE.write_text(json.dumps(state, indent=2, default=str))
|
||||
|
||||
def send_telegram(msg: str) -> None:
|
||||
"""Отправить уведомление через hermes send (если настроен TELEGRAM_HOME_CHANNEL)"""
|
||||
try:
|
||||
subprocess.run(
|
||||
["hermes", "send", "-t", "telegram", msg],
|
||||
capture_output=True, timeout=10
|
||||
)
|
||||
except Exception:
|
||||
pass # telegram не настроен — не критично
|
||||
|
||||
# ── Config ───────────────────────────────────────────────
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"check_interval": 300,
|
||||
"health_timeout": 15,
|
||||
"cooldown_minutes": 30,
|
||||
"api_key": "sk-a345af809e8a26f0693b9405344edc8adc5b5a96", # 9Router key
|
||||
"health_url": "http://localhost:20127/v1/chat/completions",
|
||||
"notify_telegram": True,
|
||||
"providers": {
|
||||
"opencode-go": {"test_model": "opencode-go/kimi-k2.7-code"},
|
||||
"grok-cli": {"test_model": "grok-cli/grok-4.5"},
|
||||
"gemini": {"test_model": "gemini/gemini-3-flash-preview"},
|
||||
"nvidia": {"test_model": "nvidia/deepseek-ai/deepseek-v4-pro"},
|
||||
},
|
||||
"fallback_matrix": {
|
||||
"worker-code": {
|
||||
"primary": "opencode-go/kimi-k2.7-code",
|
||||
"fallbacks": [
|
||||
"grok-cli/grok-4.5",
|
||||
"gemini/gemini-3-flash-preview",
|
||||
],
|
||||
},
|
||||
"worker-fast": {
|
||||
"primary": "grok-cli/grok-4.5",
|
||||
"fallbacks": [
|
||||
"opencode-go/kimi-k2.7-code",
|
||||
"gemini/gemini-3-flash-preview",
|
||||
],
|
||||
},
|
||||
"worker-research": {
|
||||
"primary": "gemini/gemini-3-flash-preview",
|
||||
"fallbacks": [
|
||||
"opencode-go/kimi-k2.7-code",
|
||||
],
|
||||
},
|
||||
"worker-review": {
|
||||
"primary": "nvidia/deepseek-ai/deepseek-v4-pro",
|
||||
"fallbacks": [
|
||||
"opencode-go/kimi-k2.7-code",
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def load_config() -> dict:
|
||||
if CONFIG_PATH.exists():
|
||||
import yaml
|
||||
with open(CONFIG_PATH) as f:
|
||||
return yaml.safe_load(f)
|
||||
return DEFAULT_CONFIG
|
||||
|
||||
# ── Health Check ─────────────────────────────────────────
|
||||
|
||||
def check_provider(provider_id: str, test_model: str, config: dict) -> str:
|
||||
"""Проверяет провайдер через 9Router health probe.
|
||||
Возвращает: ok, quota_exceeded, payment_required, error, timeout
|
||||
"""
|
||||
body = json.dumps({
|
||||
"model": test_model,
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"max_tokens": 1,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
config["health_url"],
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {config['api_key']}",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=config["health_timeout"])
|
||||
return "ok"
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
return "quota_exceeded"
|
||||
if e.code == 402:
|
||||
return "payment_required"
|
||||
if e.code in (401, 403):
|
||||
return "auth_error"
|
||||
return f"http_{e.code}"
|
||||
except Exception:
|
||||
return "timeout"
|
||||
|
||||
# ── Profile Switching ────────────────────────────────────
|
||||
|
||||
def get_current_model(profile: str) -> Optional[str]:
|
||||
"""Получить текущую модель профиля"""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["hermes", "config", "get", "model.default", "-p", profile],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
return r.stdout.strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def switch_profile_model(profile: str, model: str) -> bool:
|
||||
"""Переключить модель профиля"""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["hermes", "config", "set", "model.default", model, "-p", profile],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
return r.returncode == 0
|
||||
except Exception as e:
|
||||
log(f" ERROR switching {profile}: {e}")
|
||||
return False
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
config = load_config()
|
||||
state = load_state()
|
||||
now = datetime.now()
|
||||
|
||||
if not config.get("enabled", True):
|
||||
return
|
||||
|
||||
log("=== Supervisor check ===")
|
||||
|
||||
# 1. Проверить всех провайдеров
|
||||
provider_status = {}
|
||||
for provider_id, pconfig in config["providers"].items():
|
||||
status = check_provider(provider_id, pconfig["test_model"], config)
|
||||
provider_status[provider_id] = status
|
||||
icon = "✅" if status == "ok" else "❌"
|
||||
log(f" {icon} {provider_id}: {status}")
|
||||
|
||||
# 2. Найти профили с упавшими провайдерами
|
||||
for profile_name, matrix in config["fallback_matrix"].items():
|
||||
primary_model = matrix["primary"]
|
||||
primary_provider = primary_model.split("/")[0]
|
||||
|
||||
if provider_status.get(primary_provider, "ok") == "ok":
|
||||
continue # провайдер жив
|
||||
|
||||
# Проверить cooldown
|
||||
last_switch_ts = state["last_switch"].get(profile_name, 0)
|
||||
cooldown = config["cooldown_minutes"] * 60
|
||||
if isinstance(last_switch_ts, str):
|
||||
last_switch_ts = datetime.fromisoformat(last_switch_ts).timestamp()
|
||||
if now.timestamp() - last_switch_ts < cooldown:
|
||||
log(f" ⏳ {profile_name}: в cooldown, пропускаю")
|
||||
continue
|
||||
|
||||
# Искать рабочий fallback
|
||||
current_model = get_current_model(profile_name)
|
||||
log(f" ⚠ {profile_name}: primary {primary_model} DOWN (current: {current_model})")
|
||||
|
||||
for fallback_model in matrix["fallbacks"]:
|
||||
fb_provider = fallback_model.split("/")[0]
|
||||
if provider_status.get(fb_provider, "ok") == "ok":
|
||||
if current_model == fallback_model:
|
||||
log(f" ✓ уже на fallback {fallback_model}")
|
||||
break
|
||||
|
||||
if switch_profile_model(profile_name, fallback_model):
|
||||
incident = {
|
||||
"timestamp": now.isoformat(),
|
||||
"profile": profile_name,
|
||||
"from_model": primary_model,
|
||||
"to_model": fallback_model,
|
||||
"reason": provider_status[primary_provider],
|
||||
}
|
||||
state["incidents"].append(incident)
|
||||
state["last_switch"][profile_name] = now.isoformat()
|
||||
save_state(state)
|
||||
|
||||
msg = (
|
||||
f"🔴 Supervisor: {profile_name}\n"
|
||||
f" {primary_model} → {fallback_model}\n"
|
||||
f" Причина: {provider_status[primary_provider]}"
|
||||
)
|
||||
log(f" ✓ SWITCHED: {primary_model} → {fallback_model}")
|
||||
if config.get("notify_telegram"):
|
||||
send_telegram(msg)
|
||||
break
|
||||
else:
|
||||
log(f" ❌ Нет доступных fallback для {profile_name}!")
|
||||
|
||||
# 3. Проверить восстановление — если primary снова жив, вернуть
|
||||
for profile_name, matrix in config["fallback_matrix"].items():
|
||||
primary_model = matrix["primary"]
|
||||
primary_provider = primary_model.split("/")[0]
|
||||
current_model = get_current_model(profile_name)
|
||||
|
||||
if current_model and current_model != primary_model and provider_status.get(primary_provider) == "ok":
|
||||
# Primary восстановился — проверить 3 раза
|
||||
ok_count = 0
|
||||
for _ in range(3):
|
||||
if check_provider(primary_provider, config["providers"][primary_provider]["test_model"], config) == "ok":
|
||||
ok_count += 1
|
||||
time.sleep(2)
|
||||
|
||||
if ok_count == 3:
|
||||
if switch_profile_model(profile_name, primary_model):
|
||||
incident = {
|
||||
"timestamp": now.isoformat(),
|
||||
"profile": profile_name,
|
||||
"from_model": current_model,
|
||||
"to_model": primary_model,
|
||||
"reason": "recovery",
|
||||
}
|
||||
state["incidents"].append(incident)
|
||||
save_state(state)
|
||||
|
||||
msg = (
|
||||
f"🟢 Supervisor: {profile_name}\n"
|
||||
f" {current_model} → {primary_model}\n"
|
||||
f" Причина: primary восстановлен"
|
||||
)
|
||||
log(f" ✓ RECOVERED: {current_model} → {primary_model}")
|
||||
if config.get("notify_telegram"):
|
||||
send_telegram(msg)
|
||||
|
||||
log("=== Check complete ===\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue