diff --git a/docs/adapters/SUPERVISOR.md b/docs/adapters/SUPERVISOR.md new file mode 100644 index 0000000..941914e --- /dev/null +++ b/docs/adapters/SUPERVISOR.md @@ -0,0 +1,179 @@ +# Supervisor Agent — надзиратель провайдеров + +**Версия:** 0.1.0-draft | **Статус:** Draft + +## Назначение + +Supervisor Agent — специализированный агент-надзиратель, который: +1. Мониторит здоровье всех провайдеров/моделей +2. Обнаруживает исчерпание квот (429, 402, 403) +3. Переключает worker-профили на fallback-модели +4. Возвращает профили на основные модели после восстановления квот +5. Ведёт лог всех инцидентов и переключений + +## Архитектура + +``` +Supervisor Agent (cron: каждые 5 мин) + │ + ├─ check_provider(provider) → status + │ ├─ HTTP health probe + │ ├─ quota check (usage API) + │ └─ latency check + │ + ├─ detect_incident(status) → action + │ ├─ 429 → switch to fallback + │ ├─ 402 → switch to fallback + │ ├─ timeout → mark degraded + │ └─ recovery → switch back to primary + │ + ├─ apply_action(profile, new_model) + │ └─ hermes config set model.default -p + │ + └─ log_incident(incident) + └─ supervisor.log + Telegram notification +``` + +## Модель инцидента + +```yaml +incident: + id: uuid + timestamp: ISO8601 + provider: opencode-go|grok-cli|gemini|nvidia|ollama + model: kimi-k2.7-code + error_code: 429|402|403|timeout + action: switch_to_fallback|switch_back|alert_only + profile_affected: worker-code|worker-fast|worker-research|worker-review + old_model: opencode-go/kimi-k2.7-code + new_model: grok-cli/grok-4.5 + recovery_eta: "2026-07-21T00:00:00Z" + status: active|resolved +``` + +## Матрица переключений + +| Профиль | Primary | Fallback 1 | Fallback 2 | Last Resort | +|---------|---------|------------|------------|-------------| +| worker-code | opencode-go/kimi | grok-4.5 | gemini-3-flash | opencode-go (любая) | +| worker-fast | grok-4.5 | opencode-go/kimi | gemini-3-flash | opencode-go | +| worker-research | gemini-3-flash | opencode-go/kimi | grok-4.5 | opencode-go | +| worker-review | nvidia/deepseek-v4 | opencode-go/kimi | gemini-3-flash | opencode-go | + +## Конфигурация supervisor + +```yaml +# supervisor.yaml (рядом с Hermes config) +supervisor: + enabled: true + check_interval: 300 # секунд (5 минут) + health_timeout: 15 # секунд на health probe + cooldown_minutes: 30 # не переключать чаще чем раз в 30 мин + notify: + telegram: true + log: true + providers: + opencode-go: + health_url: http://localhost:20127/v1/models + quota_reset: "daily" # когда сбрасывается квота + grok-cli: + health_url: http://localhost:20127/v1/models + quota_reset: "monthly" + gemini: + health_url: http://localhost:20127/v1/models + quota_reset: "daily" + nvidia: + health_url: http://localhost:20127/v1/models + quota_reset: "daily" + fallback_matrix: + worker-code: + primary: opencode-go/kimi-k2.7-code + fallbacks: + - grok-cli/grok-4.5 + - gemini/gemini-3-flash-preview + worker-fast: + primary: grok-cli/grok-4.5 + fallbacks: + - opencode-go/kimi-k2.7-code + - gemini/gemini-3-flash-preview + worker-research: + primary: gemini/gemini-3-flash-preview + fallbacks: + - opencode-go/kimi-k2.7-code + worker-review: + primary: nvidia/deepseek-ai/deepseek-v4-pro + fallbacks: + - opencode-go/kimi-k2.7-code +``` + +## Алгоритм работы + +``` +1. КАЖДЫЕ check_interval секунд: + a. Для каждого провайдера: POST health_url → статус + b. Если 429/402/403 → INCIDENT + c. Если timeout × 3 → DEGRADED + +2. ПРИ ИНЦИДЕНТЕ: + a. Проверить cooldown — если последнее переключение < 30 мин назад → alert_only + b. Найти все профили с этим провайдером как primary + c. Для каждого: переключить на первый доступный fallback + d. Записать incident в лог + e. Уведомить в Telegram + +3. ПРИ ВОССТАНОВЛЕНИИ: + a. Проверить primary провайдер 3 раза с интервалом 60с + b. Если все 3 проверки OK → вернуть профили на primary + c. Обновить incident.status = resolved +``` + +## Реализация + +### Как cron-задача Hermes + +```bash +hermes cron create "*/5 * * * *" \ + --name "supervisor-health-check" \ + --script scripts/supervisor.py \ + --no_agent # script-only, не тратит токены LLM +``` + +### supervisor.py (псевдокод) + +```python +def check_provider(health_url, api_key): + r = requests.post(health_url, headers={"Authorization": f"Bearer {api_key}"}, + json={"model": "test", "messages": [{"role":"user","content":"Hi"}], "max_tokens":1}) + if r.status_code == 429: + return "quota_exceeded" + if r.status_code == 402: + return "payment_required" + if r.status_code != 200: + return "error" + return "ok" + +def switch_profile(profile, model): + subprocess.run(["hermes", "config", "set", "model.default", model, "-p", profile]) + +def main(): + for provider in config["providers"]: + status = check_provider(provider["health_url"], API_KEY) + if status != "ok": + # Найти затронутые профили и переключить + for profile, fb in config["fallback_matrix"].items(): + if fb["primary"].startswith(provider): + for fallback_model in fb["fallbacks"]: + if check_provider(...) == "ok": + switch_profile(profile, fallback_model) + notify_telegram(f"{profile}: {fb['primary']} → {fallback_model}") + break +``` + +## Интеграция с Agent Control Center + +Supervisor Agent — это Connector типа `hermes-cron` с адаптером `supervisor`. +В терминах ACC: +- **Agent:** supervisor (runtime: hermes-cron, model: none — script-only) +- **Capabilities:** provider_health_check, profile_switch, incident_logging +- **Connector:** локальный (встроен в Hermes) +- **Approvals:** не требуются (переключение модели — низкий риск) diff --git a/scripts/supervisor.py b/scripts/supervisor.py new file mode 100644 index 0000000..69d0071 --- /dev/null +++ b/scripts/supervisor.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +""" +Supervisor Agent — мониторинг и авто-переключение worker-профилей при отказе провайдеров. + +Запуск: + python scripts/supervisor.py + или как cron: hermes cron create "*/5 * * * *" --script scripts/supervisor.py --no_agent + +Конфигурация: supervisor.yaml (рядом с Hermes config) +""" + +import json +import os +import subprocess +import sys +import time +import urllib.request +import urllib.error +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +HERMES_HOME = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) +CONFIG_PATH = HERMES_HOME / "supervisor.yaml" +INCIDENT_LOG = HERMES_HOME / "logs" / "supervisor.log" +STATE_FILE = HERMES_HOME / "cache" / "supervisor_state.json" + +# ── Helpers ────────────────────────────────────────────── + +def log(msg: str) -> None: + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + line = f"[{timestamp}] {msg}" + print(line, file=sys.stderr) + INCIDENT_LOG.parent.mkdir(parents=True, exist_ok=True) + with open(INCIDENT_LOG, "a", encoding="utf-8") as f: + f.write(line + "\n") + +def load_state() -> dict: + if STATE_FILE.exists(): + return json.loads(STATE_FILE.read_text()) + return {"incidents": [], "last_switch": {}, "last_check": {}} + +def save_state(state: dict) -> None: + STATE_FILE.parent.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(json.dumps(state, indent=2, default=str)) + +def send_telegram(msg: str) -> None: + """Отправить уведомление через hermes send (если настроен TELEGRAM_HOME_CHANNEL)""" + try: + subprocess.run( + ["hermes", "send", "-t", "telegram", msg], + capture_output=True, timeout=10 + ) + except Exception: + pass # telegram не настроен — не критично + +# ── Config ─────────────────────────────────────────────── + +DEFAULT_CONFIG = { + "check_interval": 300, + "health_timeout": 15, + "cooldown_minutes": 30, + "api_key": "sk-a345af809e8a26f0693b9405344edc8adc5b5a96", # 9Router key + "health_url": "http://localhost:20127/v1/chat/completions", + "notify_telegram": True, + "providers": { + "opencode-go": {"test_model": "opencode-go/kimi-k2.7-code"}, + "grok-cli": {"test_model": "grok-cli/grok-4.5"}, + "gemini": {"test_model": "gemini/gemini-3-flash-preview"}, + "nvidia": {"test_model": "nvidia/deepseek-ai/deepseek-v4-pro"}, + }, + "fallback_matrix": { + "worker-code": { + "primary": "opencode-go/kimi-k2.7-code", + "fallbacks": [ + "grok-cli/grok-4.5", + "gemini/gemini-3-flash-preview", + ], + }, + "worker-fast": { + "primary": "grok-cli/grok-4.5", + "fallbacks": [ + "opencode-go/kimi-k2.7-code", + "gemini/gemini-3-flash-preview", + ], + }, + "worker-research": { + "primary": "gemini/gemini-3-flash-preview", + "fallbacks": [ + "opencode-go/kimi-k2.7-code", + ], + }, + "worker-review": { + "primary": "nvidia/deepseek-ai/deepseek-v4-pro", + "fallbacks": [ + "opencode-go/kimi-k2.7-code", + ], + }, + }, +} + +def load_config() -> dict: + if CONFIG_PATH.exists(): + import yaml + with open(CONFIG_PATH) as f: + return yaml.safe_load(f) + return DEFAULT_CONFIG + +# ── Health Check ───────────────────────────────────────── + +def check_provider(provider_id: str, test_model: str, config: dict) -> str: + """Проверяет провайдер через 9Router health probe. + Возвращает: ok, quota_exceeded, payment_required, error, timeout + """ + body = json.dumps({ + "model": test_model, + "messages": [{"role": "user", "content": "Hi"}], + "max_tokens": 1, + "stream": False, + }).encode() + + req = urllib.request.Request( + config["health_url"], + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {config['api_key']}", + }, + ) + + try: + resp = urllib.request.urlopen(req, timeout=config["health_timeout"]) + return "ok" + except urllib.error.HTTPError as e: + if e.code == 429: + return "quota_exceeded" + if e.code == 402: + return "payment_required" + if e.code in (401, 403): + return "auth_error" + return f"http_{e.code}" + except Exception: + return "timeout" + +# ── Profile Switching ──────────────────────────────────── + +def get_current_model(profile: str) -> Optional[str]: + """Получить текущую модель профиля""" + try: + r = subprocess.run( + ["hermes", "config", "get", "model.default", "-p", profile], + capture_output=True, text=True, timeout=10, + ) + return r.stdout.strip() + except Exception: + return None + +def switch_profile_model(profile: str, model: str) -> bool: + """Переключить модель профиля""" + try: + r = subprocess.run( + ["hermes", "config", "set", "model.default", model, "-p", profile], + capture_output=True, text=True, timeout=15, + ) + return r.returncode == 0 + except Exception as e: + log(f" ERROR switching {profile}: {e}") + return False + +# ── Main ───────────────────────────────────────────────── + +def main(): + config = load_config() + state = load_state() + now = datetime.now() + + if not config.get("enabled", True): + return + + log("=== Supervisor check ===") + + # 1. Проверить всех провайдеров + provider_status = {} + for provider_id, pconfig in config["providers"].items(): + status = check_provider(provider_id, pconfig["test_model"], config) + provider_status[provider_id] = status + icon = "✅" if status == "ok" else "❌" + log(f" {icon} {provider_id}: {status}") + + # 2. Найти профили с упавшими провайдерами + for profile_name, matrix in config["fallback_matrix"].items(): + primary_model = matrix["primary"] + primary_provider = primary_model.split("/")[0] + + if provider_status.get(primary_provider, "ok") == "ok": + continue # провайдер жив + + # Проверить cooldown + last_switch_ts = state["last_switch"].get(profile_name, 0) + cooldown = config["cooldown_minutes"] * 60 + if isinstance(last_switch_ts, str): + last_switch_ts = datetime.fromisoformat(last_switch_ts).timestamp() + if now.timestamp() - last_switch_ts < cooldown: + log(f" ⏳ {profile_name}: в cooldown, пропускаю") + continue + + # Искать рабочий fallback + current_model = get_current_model(profile_name) + log(f" ⚠ {profile_name}: primary {primary_model} DOWN (current: {current_model})") + + for fallback_model in matrix["fallbacks"]: + fb_provider = fallback_model.split("/")[0] + if provider_status.get(fb_provider, "ok") == "ok": + if current_model == fallback_model: + log(f" ✓ уже на fallback {fallback_model}") + break + + if switch_profile_model(profile_name, fallback_model): + incident = { + "timestamp": now.isoformat(), + "profile": profile_name, + "from_model": primary_model, + "to_model": fallback_model, + "reason": provider_status[primary_provider], + } + state["incidents"].append(incident) + state["last_switch"][profile_name] = now.isoformat() + save_state(state) + + msg = ( + f"🔴 Supervisor: {profile_name}\n" + f" {primary_model} → {fallback_model}\n" + f" Причина: {provider_status[primary_provider]}" + ) + log(f" ✓ SWITCHED: {primary_model} → {fallback_model}") + if config.get("notify_telegram"): + send_telegram(msg) + break + else: + log(f" ❌ Нет доступных fallback для {profile_name}!") + + # 3. Проверить восстановление — если primary снова жив, вернуть + for profile_name, matrix in config["fallback_matrix"].items(): + primary_model = matrix["primary"] + primary_provider = primary_model.split("/")[0] + current_model = get_current_model(profile_name) + + if current_model and current_model != primary_model and provider_status.get(primary_provider) == "ok": + # Primary восстановился — проверить 3 раза + ok_count = 0 + for _ in range(3): + if check_provider(primary_provider, config["providers"][primary_provider]["test_model"], config) == "ok": + ok_count += 1 + time.sleep(2) + + if ok_count == 3: + if switch_profile_model(profile_name, primary_model): + incident = { + "timestamp": now.isoformat(), + "profile": profile_name, + "from_model": current_model, + "to_model": primary_model, + "reason": "recovery", + } + state["incidents"].append(incident) + save_state(state) + + msg = ( + f"🟢 Supervisor: {profile_name}\n" + f" {current_model} → {primary_model}\n" + f" Причина: primary восстановлен" + ) + log(f" ✓ RECOVERED: {current_model} → {primary_model}") + if config.get("notify_telegram"): + send_telegram(msg) + + log("=== Check complete ===\n") + +if __name__ == "__main__": + main()