6cb0d7c636
Владелец спросил, откуда 25 каталогов профилей, когда заводил единицы. Проверено исполнением: обращение к get_profile_dir создавало каталог. Спросили четыре пути — появились четыре каталога. В коде зашит список «стандартных» слотов — ag-orch-primary, ag-orch-fallback, ag-1..ag-20, ag-w1..ag-w10 — в двух местах: agy_subprocess и model_discovery_service. Любой обход этого списка материализовал их все. Отсюда ag-w1..ag-w4, ag-cold-*, ag-spare-* — владелец их не создавал. Спросить, где профиль жил бы, и завести его — разные действия. Создание теперь запрашивается явно, create=True, и это делают только те, кто действительно заводит профиль. Тесты и сценарий проверки, опиравшиеся на побочное создание, приведены к новому поведению: они проверяют изоляцию пути, а не то, что каталог возник сам. 669 passed, 2 skipped; ruff чисто; релизный гейт пройден. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
263 lines
11 KiB
Python
263 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Automated verification suite for Hermes Multi-Provider Account Router."""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
from contextlib import ExitStack
|
||
from unittest.mock import patch
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||
for p in [
|
||
REPO_ROOT / "src",
|
||
REPO_ROOT / "plugins" / "antigravity-provider" / "src",
|
||
Path(os.environ.get("LOCALAPPDATA", "")) / "hermes" / "plugins" / "antigravity-provider" / "src",
|
||
]:
|
||
if p.is_dir() and str(p) not in sys.path:
|
||
sys.path.insert(0, str(p))
|
||
|
||
from antigravity_provider.router.router_config import (
|
||
RolePolicy,
|
||
RouterConfig,
|
||
RouterProfileConfig,
|
||
get_default_router_config,
|
||
load_router_config,
|
||
)
|
||
from antigravity_provider.router.health_tracker import (
|
||
HEALTHY,
|
||
QUOTA_EXHAUSTED,
|
||
RATE_LIMITED,
|
||
HealthTracker,
|
||
extract_model_family,
|
||
)
|
||
from antigravity_provider.router.session_affinity import LeaseManager, SessionAffinityTracker
|
||
from antigravity_provider.router.router_engine import RouterEngine, get_router_engine
|
||
from antigravity_provider.router.adapters.antigravity_adapter import AntigravityAdapter, get_profile_env_dir
|
||
from antigravity_provider.router.adapters.codex_adapter import CodexAdapter
|
||
from antigravity_provider.router.adapters.opencode_adapter import OpenCodeGoAdapter
|
||
from antigravity_provider.router.adapters.claude_adapter import ClaudeAdapter
|
||
from antigravity_provider.router.adapters.grok_adapter import GrokAdapter
|
||
from antigravity_provider.router.adapters.local_adapter import LocalLLMAdapter
|
||
|
||
|
||
def run_checks() -> int:
|
||
print("=" * 70)
|
||
print("HERMES MULTI-PROVIDER ACCOUNT ROUTER: AUTOMATED VERIFICATION")
|
||
print("=" * 70)
|
||
|
||
passed = 0
|
||
total = 10
|
||
|
||
# 1. Config inventory
|
||
print("1. Checking profile inventory and provider counts...")
|
||
config = load_router_config()
|
||
if config.profiles:
|
||
counts = {}
|
||
for prof in config.profiles.values():
|
||
counts[prof.provider] = counts.get(prof.provider, 0) + 1
|
||
summary = ", ".join(f"{prov}: {n}" for prov, n in sorted(counts.items()))
|
||
print(f" [PASS] Профилей: {len(config.profiles)} ({summary})")
|
||
else:
|
||
print(" [PASS] Чистая конфигурация по умолчанию (0 профилей, добавление по мере подключения)")
|
||
passed += 1
|
||
|
||
# 2. Role Fallback Chains
|
||
print("2. Checking role fallback policies...")
|
||
assert config.roles, "В конфигурации нет ни одной роли"
|
||
try:
|
||
from antigravity_provider.router.role_registry import RoleRegistry
|
||
|
||
orchestrating_role = RoleRegistry.resolve_canonical_role("orchestrator")
|
||
except Exception:
|
||
orchestrating_role = "manager"
|
||
assert orchestrating_role in config.roles, (
|
||
f"Оркестрирующая роль {orchestrating_role!r} отсутствует; есть: {sorted(config.roles)}"
|
||
)
|
||
empty_chains = []
|
||
for role_name, policy in config.roles.items():
|
||
chain = policy.preferred_chain or []
|
||
if not chain:
|
||
empty_chains.append(role_name)
|
||
for pid in chain:
|
||
assert pid in config.profiles, (
|
||
f"Роль {role_name} ссылается на несуществующий профиль {pid}"
|
||
)
|
||
if empty_chains:
|
||
print(f" [INFO] Роли с пустыми цепочками (чистый старт / не назначены): {', '.join(sorted(empty_chains))}")
|
||
print(f" [PASS] Все {len(config.roles)} ролей валидны и ссылаются только на существующие профили")
|
||
passed += 1
|
||
|
||
# 3. Model family extraction
|
||
print("3. Checking model family extraction...")
|
||
assert extract_model_family("gemini-3.7-flash") == "gemini"
|
||
assert extract_model_family("claude-sonnet-4-6") == "claude"
|
||
assert extract_model_family("gpt-4o") == "gpt"
|
||
assert extract_model_family("kimi-k2.7-code") == "kimi"
|
||
assert extract_model_family("deepseek-v4-pro") == "deepseek"
|
||
print(" [PASS] Model family parsing correct")
|
||
passed += 1
|
||
|
||
# 4. Health state transitions & simulated quota
|
||
print("4. Checking health states and simulated quota...")
|
||
state_file = REPO_ROOT / "tests" / "fixtures" / "scratch_state.json"
|
||
tracker = HealthTracker(state_file=state_file)
|
||
tracker.clear_cooldown()
|
||
assert tracker.is_healthy("codex-orch") is True
|
||
tracker.simulate_quota("codex-orch", duration=300)
|
||
assert tracker.is_healthy("codex-orch") is False
|
||
tracker.clear_cooldown("codex-orch")
|
||
assert tracker.is_healthy("codex-orch") is True
|
||
if state_file.exists():
|
||
state_file.unlink()
|
||
print(" [PASS] Quota simulation and recovery verified")
|
||
passed += 1
|
||
|
||
# 5. Session Affinity Retention
|
||
print("5. Checking session affinity engine...")
|
||
affinity = SessionAffinityTracker()
|
||
affinity.set_affinity("session-test-01", "orchestrator", "codex-orch", "gpt-4o")
|
||
rec = affinity.get_affinity("session-test-01")
|
||
assert rec and rec.profile_id == "codex-orch"
|
||
affinity.set_affinity("session-test-01", "orchestrator", "ag-orch-fallback", "gemini-3.7-flash")
|
||
assert affinity.get_affinity("session-test-01").profile_id == "ag-orch-fallback"
|
||
print(" [PASS] Session affinity tracking & update verified")
|
||
passed += 1
|
||
|
||
# 6. Concurrency Leases
|
||
print("6. Checking concurrency lease limits...")
|
||
leases = LeaseManager()
|
||
assert leases.acquire("ag-w1", max_concurrency=1) is True
|
||
assert leases.acquire("ag-w1", max_concurrency=1) is False
|
||
leases.release("ag-w1")
|
||
assert leases.acquire("ag-w1", max_concurrency=1) is True
|
||
leases.release("ag-w1")
|
||
print(" [PASS] Concurrency leases correctly enforced")
|
||
passed += 1
|
||
|
||
# 7. Antigravity profile isolation
|
||
print("7. Checking Antigravity profile environment directory isolation...")
|
||
# Проверяем изоляцию пути, а не побочное создание каталога: запрос пути
|
||
# каталогов больше не плодит, иначе любая проверка засоряла бы диск
|
||
# десятком пустых слотов.
|
||
pdir = get_profile_env_dir("ag-w2")
|
||
assert "ag-w2" in str(pdir)
|
||
assert "agy_profiles" in str(pdir)
|
||
assert not pdir.exists(), "запрос пути не должен создавать каталог"
|
||
print(f" [PASS] Profile directory isolated at {pdir} (не создан)")
|
||
passed += 1
|
||
|
||
# 8. Error classification
|
||
print("8. Checking provider error classification...")
|
||
ag_adapter = AntigravityAdapter()
|
||
ag_err = ag_adapter.classify_error(RuntimeError("RESOURCE_EXHAUSTED: Individual quota reached for gemini"))
|
||
assert ag_err.category == "quota-exhausted"
|
||
|
||
codex_adapter = CodexAdapter()
|
||
codex_err = codex_adapter.classify_error(RuntimeError("HTTP Error 429: Rate limit reached for tokens per minute"))
|
||
assert codex_err.category == "rate-limited"
|
||
|
||
opengo_adapter = OpenCodeGoAdapter()
|
||
opengo_err = opengo_adapter.classify_error(RuntimeError("HTTP Error 401: Invalid API Key"))
|
||
assert opengo_err.category == "auth-required"
|
||
print(" [PASS] Error classifications for all 3 providers verified")
|
||
passed += 1
|
||
|
||
# 9. Full failover execution loop
|
||
print("9. Checking full failover execution loop...")
|
||
full_chain = list(config.roles.get(orchestrating_role, RolePolicy(role_name=orchestrating_role)).preferred_chain)
|
||
max_attempts = getattr(config.roles.get(orchestrating_role, RolePolicy(role_name=orchestrating_role)), "max_failover_attempts", 0) or len(full_chain)
|
||
chain = full_chain[:max_attempts]
|
||
|
||
adapter_by_provider = {
|
||
"openai-codex": CodexAdapter,
|
||
"antigravity": AntigravityAdapter,
|
||
"opencode-go": OpenCodeGoAdapter,
|
||
"claude": ClaudeAdapter,
|
||
"grok": GrokAdapter,
|
||
"local": LocalLLMAdapter,
|
||
}
|
||
|
||
if len(chain) >= 2 and all(pid in config.profiles for pid in chain):
|
||
test_engine = RouterEngine(config=config)
|
||
test_engine.health.clear_cooldown()
|
||
test_role = orchestrating_role
|
||
test_chain = chain
|
||
test_profiles = config.profiles
|
||
else:
|
||
synth_config = RouterConfig(
|
||
enabled=True,
|
||
default_role="manager",
|
||
roles={
|
||
"manager": RolePolicy(
|
||
role_name="manager",
|
||
preferred_chain=["synth-codex", "synth-ag"],
|
||
max_failover_attempts=2,
|
||
)
|
||
},
|
||
profiles={
|
||
"synth-codex": RouterProfileConfig(
|
||
profile_id="synth-codex",
|
||
provider="openai-codex",
|
||
capabilities=["coding", "orchestrator"],
|
||
),
|
||
"synth-ag": RouterProfileConfig(
|
||
profile_id="synth-ag",
|
||
provider="antigravity",
|
||
capabilities=["coding", "orchestrator"],
|
||
),
|
||
},
|
||
)
|
||
test_engine = RouterEngine(config=synth_config)
|
||
test_engine.health.clear_cooldown()
|
||
test_role = "manager"
|
||
test_chain = ["synth-codex", "synth-ag"]
|
||
test_profiles = synth_config.profiles
|
||
|
||
last_pid = test_chain[-1]
|
||
last_provider = test_profiles[last_pid].provider
|
||
|
||
expected = {"id": "ok", "choices": [{"message": {"role": "assistant", "content": "from-last-in-chain"}}]}
|
||
failing = {cls for pid in test_chain[:-1]
|
||
if (cls := adapter_by_provider.get(test_profiles[pid].provider)) is not None}
|
||
winner = adapter_by_provider.get(last_provider)
|
||
|
||
if winner is None or winner in failing:
|
||
print(f" [SKIP] Последний профиль цепочки ({last_pid}, {last_provider}) "
|
||
"не покрыт заглушками адаптеров")
|
||
else:
|
||
with ExitStack() as stack:
|
||
for cls in failing:
|
||
stack.enter_context(patch.object(cls, "invoke", side_effect=RuntimeError("Insufficient quota")))
|
||
stack.enter_context(patch.object(winner, "invoke", return_value=expected))
|
||
res = test_engine.route_request(
|
||
{"messages": [{"role": "user", "content": "test"}]},
|
||
role=test_role,
|
||
session_id="verify-failover",
|
||
)
|
||
assert res["choices"][0]["message"]["content"] == "from-last-in-chain", (
|
||
f"Отказоустойчивость не дошла до последнего профиля цепочки: {res}"
|
||
)
|
||
assert res["router_metadata"]["profile_id"] == last_pid, (
|
||
f"Ожидался профиль {last_pid}, получен {res['router_metadata']['profile_id']}"
|
||
)
|
||
print(f" [PASS] Отказоустойчивость прошла цепочку {' -> '.join(test_chain)}")
|
||
passed += 1
|
||
|
||
# 10. Passthrough & Graceful fallback
|
||
print("10. Checking disabled router passthrough...")
|
||
disabled_config = get_default_router_config()
|
||
disabled_config.enabled = False
|
||
assert disabled_config.enabled is False
|
||
print(" [PASS] Router clean bypass mode verified")
|
||
passed += 1
|
||
|
||
print("-" * 70)
|
||
print(f"VERIFICATION COMPLETE: {passed}/{total} CHECKS PASSED (0 errors, 0 warnings)")
|
||
print("=" * 70)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(run_checks())
|