feat(router): profile auth self-healing, honest model validation, and manual model refresh (A23)
This commit is contained in:
parent
9c6a4e8f6d
commit
1e1b81665b
9 changed files with 584 additions and 9 deletions
|
|
@ -139,6 +139,28 @@ def do_save_settings(settings: Dict[str, Any]) -> Tuple[bool, str]:
|
|||
return True, "Настройки сохранены"
|
||||
|
||||
|
||||
def _model_matches_discovered(model: str, discovered: list[str]) -> bool:
|
||||
if model in discovered:
|
||||
return True
|
||||
model_short = model.split("/")[-1]
|
||||
for d in discovered:
|
||||
if d == model or d == model_short:
|
||||
return True
|
||||
d_short = d.split("/")[-1]
|
||||
if d_short == model or d_short == model_short:
|
||||
return True
|
||||
if d.startswith(model + "-") or d.startswith(model + ":"):
|
||||
return True
|
||||
if d_short.startswith(model_short + "-") or d_short.startswith(model_short + ":"):
|
||||
return True
|
||||
import re
|
||||
d_base = re.sub(r"-(high|medium|low|none|thought|thinking)(?:-(high|medium|low|none))?$", "", d_short)
|
||||
model_base = re.sub(r"-(high|medium|low|none|thought|thinking)(?:-(high|medium|low|none))?$", "", model_short)
|
||||
if d_base == model_short or d_base == model_base or d_short == model_base:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def do_set_model(profile_id: str, model: str, role_id: Optional[str] = None) -> Tuple[bool, str]:
|
||||
if not model or not str(model).strip() or str(model).strip() == "Список моделей ещё не получен":
|
||||
return False, "Не указана модель для установки"
|
||||
|
|
@ -160,11 +182,27 @@ def do_set_model(profile_id: str, model: str, role_id: Optional[str] = None) ->
|
|||
from antigravity_provider.router.model_registry import ModelRegistry
|
||||
|
||||
discovered = ModelDiscoveryService.get().get_models(provider)
|
||||
if discovered is None:
|
||||
try:
|
||||
discovered = ModelDiscoveryService.get().discover_models_sync(provider, timeout=5.0)
|
||||
except Exception:
|
||||
discovered = None
|
||||
|
||||
canonical = [m.model_id for m in ModelRegistry.get().list_models(provider=provider)]
|
||||
canonical_short = [m.split("/")[-1] for m in canonical]
|
||||
|
||||
if discovered is not None:
|
||||
if model not in discovered and model not in canonical and model not in canonical_short:
|
||||
is_canonical = (
|
||||
model in canonical
|
||||
or model in canonical_short
|
||||
or _model_matches_discovered(model, canonical)
|
||||
)
|
||||
|
||||
if not discovered:
|
||||
if not is_canonical:
|
||||
return False, f"Кэш моделей для провайдера '{provider}' пуст, а модель '{model}' не найдена в списке известных моделей."
|
||||
else:
|
||||
matches_discovered = _model_matches_discovered(model, discovered)
|
||||
if not matches_discovered and not is_canonical:
|
||||
return False, f"Модель '{model}' отсутствует в списке обнаруженных моделей провайдера '{provider}'"
|
||||
|
||||
updated = load_router_config()
|
||||
|
|
@ -257,6 +295,24 @@ class ActionExecutor:
|
|||
else:
|
||||
HermesRefreshScheduler.get().trigger_refresh_account(prov, pid)
|
||||
return {'ok': True, 'message': 'Успешно'}
|
||||
|
||||
elif action == 'refresh_models':
|
||||
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||
service = ModelDiscoveryService.get()
|
||||
if prov:
|
||||
if async_runner:
|
||||
async_runner(lambda: service.discover_models_sync(prov), 'RefreshModels')
|
||||
return {'ok': True, 'message': 'запущено'}
|
||||
else:
|
||||
res = service.discover_models_sync(prov)
|
||||
return {'ok': True, 'message': 'Успешно', 'data': res}
|
||||
else:
|
||||
if async_runner:
|
||||
async_runner(lambda: service.refresh_all_async(), 'RefreshAllModels')
|
||||
return {'ok': True, 'message': 'запущено'}
|
||||
else:
|
||||
service.refresh_all_async()
|
||||
return {'ok': True, 'message': 'запущено'}
|
||||
|
||||
elif action == 'save_settings':
|
||||
ok, msg = do_save_settings(data)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ class ProfileHealthRecord:
|
|||
last_used: Optional[float] = None
|
||||
last_success: Optional[float] = None
|
||||
last_error: Optional[str] = None
|
||||
auth_error_at: Optional[float] = None
|
||||
simulated: bool = False
|
||||
|
||||
|
||||
|
|
@ -76,12 +77,18 @@ class _FileLock:
|
|||
self._fd = os.open(str(self.lock_path), os.O_CREAT | os.O_RDWR)
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1)
|
||||
msvcrt.locking(self._fd, msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
fcntl.flock(self._fd, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
pass
|
||||
if self._fd is not None:
|
||||
try:
|
||||
os.close(self._fd)
|
||||
except Exception:
|
||||
pass
|
||||
self._fd = None
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
|
|
@ -118,6 +125,99 @@ class HealthTracker:
|
|||
self._profiles: dict[str, ProfileHealthRecord] = {}
|
||||
self._load_state()
|
||||
|
||||
try:
|
||||
from antigravity_provider.router.event_bus import (
|
||||
EVENT_ACCOUNT_ADDED,
|
||||
EVENT_ACCOUNT_AUTH_CHANGED,
|
||||
EventBus,
|
||||
)
|
||||
EventBus.get().subscribe(EVENT_ACCOUNT_ADDED, self._on_account_event)
|
||||
EventBus.get().subscribe(EVENT_ACCOUNT_AUTH_CHANGED, self._on_account_event)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _on_account_event(self, _event_name: str, payload: Any) -> None:
|
||||
"""Handle account lifecycle events by recovering profile from AUTH_REQUIRED state."""
|
||||
profile_id = None
|
||||
if isinstance(payload, dict):
|
||||
profile_id = payload.get("profile_id")
|
||||
elif hasattr(payload, "profile_id"):
|
||||
profile_id = getattr(payload, "profile_id")
|
||||
if not profile_id:
|
||||
return
|
||||
profile_id = str(profile_id)
|
||||
with self._lock:
|
||||
if profile_id in self._profiles:
|
||||
rec = self._profiles[profile_id]
|
||||
rec.overall_state = HEALTHY
|
||||
rec.last_error = None
|
||||
rec.auth_error_at = None
|
||||
for frec in rec.families.values():
|
||||
if frec.state == AUTH_REQUIRED:
|
||||
frec.state = HEALTHY
|
||||
frec.reset_at = None
|
||||
frec.reason = None
|
||||
frec.last_error = None
|
||||
self._save_state()
|
||||
|
||||
def _find_profile_auth_files(self, profile_id: str) -> list[Path]:
|
||||
"""Locate credentials and auth files for the specified profile."""
|
||||
files: list[Path] = []
|
||||
try:
|
||||
pdir = paths.get_profile_dir(profile_id)
|
||||
for cand in (pdir / "auth.json", pdir / ".gemini" / "oauth_creds.json", pdir / "oauth_creds.json"):
|
||||
if cand.is_file():
|
||||
files.append(cand)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
hermes_home = paths.get_hermes_home()
|
||||
if hermes_home.is_dir():
|
||||
for pdir in hermes_home.glob(f"*_profiles/{profile_id}"):
|
||||
if pdir.is_dir():
|
||||
for cand in (pdir / "auth.json", pdir / ".gemini" / "oauth_creds.json", pdir / "oauth_creds.json"):
|
||||
if cand.is_file() and cand not in files:
|
||||
files.append(cand)
|
||||
except Exception:
|
||||
pass
|
||||
return files
|
||||
|
||||
def _check_and_recover_auth(self, record: ProfileHealthRecord) -> bool:
|
||||
"""Check if auth files exist, are valid, and were modified after an auth failure."""
|
||||
auth_files = self._find_profile_auth_files(record.profile_id)
|
||||
if not auth_files:
|
||||
return False
|
||||
|
||||
recovered = False
|
||||
for f in auth_files:
|
||||
try:
|
||||
stat = f.stat()
|
||||
if stat.st_size == 0:
|
||||
continue
|
||||
if record.auth_error_at is not None and stat.st_mtime <= record.auth_error_at:
|
||||
continue
|
||||
content = json.loads(f.read_text(encoding="utf-8"))
|
||||
if isinstance(content, dict) and len(content) > 0:
|
||||
recovered = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if recovered:
|
||||
record.overall_state = HEALTHY
|
||||
record.last_error = None
|
||||
record.auth_error_at = None
|
||||
for frec in record.families.values():
|
||||
if frec.state == AUTH_REQUIRED:
|
||||
frec.state = HEALTHY
|
||||
frec.reset_at = None
|
||||
frec.reason = None
|
||||
frec.last_error = None
|
||||
self._save_state()
|
||||
return True
|
||||
return False
|
||||
|
||||
def _load_state(self) -> None:
|
||||
if not self.state_file.is_file():
|
||||
return
|
||||
|
|
@ -130,6 +230,7 @@ class HealthTracker:
|
|||
last_used=pdata.get("last_used"),
|
||||
last_success=pdata.get("last_success"),
|
||||
last_error=pdata.get("last_error"),
|
||||
auth_error_at=pdata.get("auth_error_at"),
|
||||
simulated=pdata.get("simulated", False),
|
||||
)
|
||||
for fname, fdata in pdata.get("families", {}).items():
|
||||
|
|
@ -158,6 +259,7 @@ class HealthTracker:
|
|||
"last_used": precord.last_used,
|
||||
"last_success": precord.last_success,
|
||||
"last_error": precord.last_error,
|
||||
"auth_error_at": precord.auth_error_at,
|
||||
"simulated": precord.simulated,
|
||||
"families": {},
|
||||
}
|
||||
|
|
@ -211,6 +313,9 @@ class HealthTracker:
|
|||
if record.overall_state == DISABLED:
|
||||
return False
|
||||
|
||||
if record.overall_state == AUTH_REQUIRED:
|
||||
self._check_and_recover_auth(record)
|
||||
|
||||
# Check profile-level default family
|
||||
if "default" in record.families:
|
||||
def_rec = record.families["default"]
|
||||
|
|
@ -257,6 +362,7 @@ class HealthTracker:
|
|||
record.last_used = now
|
||||
record.last_success = now
|
||||
record.overall_state = HEALTHY
|
||||
record.auth_error_at = None
|
||||
record.simulated = False
|
||||
|
||||
family = extract_model_family(model_name)
|
||||
|
|
@ -378,6 +484,7 @@ class HealthTracker:
|
|||
record = self.get_or_create(profile_id)
|
||||
record.overall_state = AUTH_REQUIRED
|
||||
record.last_error = reason
|
||||
record.auth_error_at = time.time()
|
||||
self._save_state()
|
||||
|
||||
def clear_cooldown(self, profile_id: Optional[str] = None, model_name: Optional[str] = None) -> None:
|
||||
|
|
@ -385,10 +492,14 @@ class HealthTracker:
|
|||
if profile_id is None:
|
||||
for rec in self._profiles.values():
|
||||
rec.overall_state = HEALTHY
|
||||
rec.last_error = None
|
||||
rec.auth_error_at = None
|
||||
rec.simulated = False
|
||||
for frec in rec.families.values():
|
||||
frec.state = HEALTHY
|
||||
frec.reset_at = None
|
||||
frec.reason = None
|
||||
frec.last_error = None
|
||||
frec.simulated = False
|
||||
self._save_state()
|
||||
return
|
||||
|
|
@ -397,16 +508,22 @@ class HealthTracker:
|
|||
return
|
||||
record = self._profiles[profile_id]
|
||||
record.overall_state = HEALTHY
|
||||
record.last_error = None
|
||||
record.auth_error_at = None
|
||||
record.simulated = False
|
||||
if model_name:
|
||||
family = extract_model_family(model_name)
|
||||
if family in record.families:
|
||||
record.families[family].state = HEALTHY
|
||||
record.families[family].reset_at = None
|
||||
record.families[family].reason = None
|
||||
record.families[family].last_error = None
|
||||
record.families[family].simulated = False
|
||||
else:
|
||||
for frec in record.families.values():
|
||||
frec.state = HEALTHY
|
||||
frec.reset_at = None
|
||||
frec.reason = None
|
||||
frec.last_error = None
|
||||
frec.simulated = False
|
||||
self._save_state()
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ class ModelDiscoveryService:
|
|||
return list(entry["models"]) if entry and "models" in entry else None
|
||||
|
||||
models = result_holder[0]
|
||||
if models is not None:
|
||||
if models:
|
||||
with self._cache_lock:
|
||||
self._cache[provider.lower()] = {
|
||||
"models": models,
|
||||
|
|
@ -208,7 +208,7 @@ class ModelDiscoveryService:
|
|||
logger.info("Discovered %d models for provider '%s': %s", len(models), provider, models)
|
||||
return models
|
||||
|
||||
# If probe returned None, retain existing cache if any
|
||||
# If probe returned None or empty, retain existing cache if any
|
||||
with self._cache_lock:
|
||||
entry = self._cache.get(provider.lower())
|
||||
return list(entry["models"]) if entry and "models" in entry else None
|
||||
|
|
|
|||
|
|
@ -216,6 +216,51 @@ class ModelRegistry:
|
|||
quota_bucket="antigravity.gemini",
|
||||
quality_tier=3,
|
||||
),
|
||||
ModelDescriptor(
|
||||
model_id="google-antigravity/gemini-3.1-pro",
|
||||
display_name="Gemini 3.1 Pro",
|
||||
provider="antigravity",
|
||||
family="gemini",
|
||||
capabilities=["coding", "reasoning", "tools", "structured_output", "long_context", "planning", "security_analysis"],
|
||||
context_window=1000000,
|
||||
supports_tools=True,
|
||||
supports_reasoning=True,
|
||||
latency_class="medium",
|
||||
cost_input_per_m=1.25,
|
||||
cost_output_per_m=5.0,
|
||||
quota_bucket="antigravity.gemini",
|
||||
quality_tier=5,
|
||||
),
|
||||
ModelDescriptor(
|
||||
model_id="google-antigravity/gemini-3.7-flash",
|
||||
display_name="Gemini 3.7 Flash",
|
||||
provider="antigravity",
|
||||
family="gemini",
|
||||
capabilities=["coding", "reasoning", "tools", "structured_output", "classification", "routing", "long_context"],
|
||||
context_window=1000000,
|
||||
supports_tools=True,
|
||||
supports_reasoning=True,
|
||||
latency_class="ultra_low",
|
||||
cost_input_per_m=0.15,
|
||||
cost_output_per_m=0.6,
|
||||
quota_bucket="antigravity.gemini",
|
||||
quality_tier=4,
|
||||
),
|
||||
ModelDescriptor(
|
||||
model_id="google-antigravity/gemini-3.7-pro",
|
||||
display_name="Gemini 3.7 Pro",
|
||||
provider="antigravity",
|
||||
family="gemini",
|
||||
capabilities=["coding", "reasoning", "tools", "structured_output", "long_context", "planning", "security_analysis"],
|
||||
context_window=1000000,
|
||||
supports_tools=True,
|
||||
supports_reasoning=True,
|
||||
latency_class="medium",
|
||||
cost_input_per_m=1.25,
|
||||
cost_output_per_m=5.0,
|
||||
quota_bucket="antigravity.gemini",
|
||||
quality_tier=5,
|
||||
),
|
||||
# Google Antigravity (Claude family inside AGY)
|
||||
ModelDescriptor(
|
||||
model_id="google-antigravity/claude-3-7-sonnet",
|
||||
|
|
|
|||
|
|
@ -24,6 +24,13 @@ def _service() -> Any:
|
|||
try:
|
||||
from antigravity_provider.router.model_discovery import ModelDiscoveryService
|
||||
|
||||
getter = getattr(ModelDiscoveryService, "get", None)
|
||||
return getter() if callable(getter) else ModelDiscoveryService()
|
||||
except (ImportError, AttributeError, TypeError):
|
||||
pass
|
||||
try:
|
||||
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||
|
||||
getter = getattr(ModelDiscoveryService, "get", None)
|
||||
return getter() if callable(getter) else ModelDiscoveryService()
|
||||
except (ImportError, AttributeError, TypeError):
|
||||
|
|
@ -80,7 +87,7 @@ def refresh_models_async(provider: str, on_complete: Callable[[CachedModels], No
|
|||
if service is None:
|
||||
on_complete(get_cached_models(provider))
|
||||
return False
|
||||
for name in ("refresh_provider_async", "refresh_async", "discover_async"):
|
||||
for name in ("refresh_models_async", "refresh_models", "refresh_provider_async", "refresh_async", "discover_async"):
|
||||
method = getattr(service, name, None)
|
||||
if not callable(method):
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -703,6 +703,16 @@ class UnifiedHealthService:
|
|||
for p in profs:
|
||||
for m in p.preferred_models:
|
||||
models_set.add(m)
|
||||
|
||||
# Add discovered models from ModelDiscoveryService cache
|
||||
try:
|
||||
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||
cached = ModelDiscoveryService.get().get_cached(prov_id)
|
||||
if cached and cached.get("has_cache") and cached.get("models"):
|
||||
for m in cached["models"]:
|
||||
models_set.add(m)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
summaries.append(ProviderSummary(
|
||||
provider_id=prov_id,
|
||||
|
|
|
|||
|
|
@ -1375,7 +1375,7 @@ async function handleSaveRoleModel(roleId, profileId) {
|
|||
|
||||
async function handleRefreshProviderModels(providerId, profileId = null) {
|
||||
showToast(`Запрос списка моделей для ${providerId}...`, 'info');
|
||||
const res = await executeAction('refresh_data', { provider: providerId });
|
||||
const res = await executeAction('refresh_models', { provider: providerId });
|
||||
if (res.ok) {
|
||||
showToast('Запрос обновления моделей отправлен', 'success');
|
||||
if (profileId) {
|
||||
|
|
|
|||
178
tests/test_health_auth_recovery.py
Normal file
178
tests/test_health_auth_recovery.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
"""Tests for health tracker and router profile self-recovery from AUTH_REQUIRED state."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from antigravity_provider.router.adapters.antigravity_adapter import AntigravityAdapter
|
||||
from antigravity_provider.router.health_tracker import (
|
||||
AUTH_REQUIRED,
|
||||
HEALTHY,
|
||||
HealthTracker,
|
||||
)
|
||||
from antigravity_provider.router.profile_manager import ProfileAuthManager
|
||||
from antigravity_provider.router.router_config import (
|
||||
RolePolicy,
|
||||
RouterConfig,
|
||||
RouterProfileConfig,
|
||||
)
|
||||
from antigravity_provider.router.router_engine import RouterEngine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_env(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
state_file = tmp_path / "router_state.json"
|
||||
ht = HealthTracker(state_file=state_file)
|
||||
return tmp_path, ht
|
||||
|
||||
|
||||
def test_auth_required_skipped_in_routing(isolated_env):
|
||||
tmp_path, ht = isolated_env
|
||||
|
||||
# Configure 2 profiles for role 'orchestrator'
|
||||
cfg = RouterConfig(
|
||||
profiles={
|
||||
"ag-orch-1": RouterProfileConfig(
|
||||
profile_id="ag-orch-1",
|
||||
provider="antigravity",
|
||||
enabled=True,
|
||||
),
|
||||
"ag-orch-2": RouterProfileConfig(
|
||||
profile_id="ag-orch-2",
|
||||
provider="antigravity",
|
||||
enabled=True,
|
||||
),
|
||||
},
|
||||
roles={
|
||||
"orchestrator": RolePolicy(
|
||||
role_name="orchestrator",
|
||||
preferred_chain=["ag-orch-1", "ag-orch-2"],
|
||||
),
|
||||
},
|
||||
default_role="orchestrator",
|
||||
)
|
||||
|
||||
engine = RouterEngine(config=cfg, health=ht)
|
||||
|
||||
def _mock_invoke(*args, **kwargs):
|
||||
return {"id": "res-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]}
|
||||
|
||||
# Initial state: ag-orch-1 is healthy and selected
|
||||
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-1"
|
||||
|
||||
# Mark ag-orch-1 as AUTH_REQUIRED
|
||||
ht.mark_auth_required("ag-orch-1", reason="401 Unauthorized token expired")
|
||||
assert not ht.is_healthy("ag-orch-1")
|
||||
|
||||
# Routing should skip ag-orch-1 and route to ag-orch-2
|
||||
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-2"
|
||||
|
||||
|
||||
def test_save_profile_auth_auto_recovery(isolated_env):
|
||||
tmp_path, ht = isolated_env
|
||||
|
||||
cfg = RouterConfig(
|
||||
profiles={
|
||||
"ag-orch-1": RouterProfileConfig(
|
||||
profile_id="ag-orch-1",
|
||||
provider="antigravity",
|
||||
enabled=True,
|
||||
),
|
||||
"ag-orch-2": RouterProfileConfig(
|
||||
profile_id="ag-orch-2",
|
||||
provider="antigravity",
|
||||
enabled=True,
|
||||
),
|
||||
},
|
||||
roles={
|
||||
"orchestrator": RolePolicy(
|
||||
role_name="orchestrator",
|
||||
preferred_chain=["ag-orch-1", "ag-orch-2"],
|
||||
),
|
||||
},
|
||||
default_role="orchestrator",
|
||||
)
|
||||
engine = RouterEngine(config=cfg, health=ht)
|
||||
|
||||
def _mock_invoke(*args, **kwargs):
|
||||
return {"id": "res-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]}
|
||||
|
||||
# Mark ag-orch-1 as AUTH_REQUIRED
|
||||
ht.mark_auth_required("ag-orch-1", reason="Auth token rejected")
|
||||
rec = ht.get_or_create("ag-orch-1")
|
||||
assert rec.overall_state == AUTH_REQUIRED
|
||||
assert rec.last_error == "Auth token rejected"
|
||||
assert not ht.is_healthy("ag-orch-1")
|
||||
|
||||
# Saving credentials via ProfileAuthManager publishes event and auto-recovers health
|
||||
ProfileAuthManager.save_profile_auth(
|
||||
"antigravity",
|
||||
"ag-orch-1",
|
||||
{"token": {"access_token": "valid_tok_123", "refresh_token": "ref_123"}},
|
||||
)
|
||||
|
||||
# State is automatically restored to HEALTHY without manual clear_cooldown
|
||||
rec = ht.get_or_create("ag-orch-1")
|
||||
assert rec.overall_state == HEALTHY
|
||||
assert rec.last_error is None
|
||||
assert ht.is_healthy("ag-orch-1")
|
||||
|
||||
# Router now routes back to primary profile ag-orch-1
|
||||
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-1"
|
||||
|
||||
|
||||
def test_auth_file_mtime_recovery_in_is_healthy(isolated_env):
|
||||
tmp_path, ht = isolated_env
|
||||
|
||||
# Setup profile directory and initial auth file
|
||||
pdir = tmp_path / "agy_profiles" / "ag-test-1"
|
||||
pdir.mkdir(parents=True, exist_ok=True)
|
||||
auth_file = pdir / "auth.json"
|
||||
auth_file.write_text(json.dumps({"token": "old"}), encoding="utf-8")
|
||||
time.sleep(0.05)
|
||||
|
||||
# Mark AUTH_REQUIRED at a timestamp
|
||||
ht.mark_auth_required("ag-test-1", reason="OAuth error")
|
||||
rec = ht.get_or_create("ag-test-1")
|
||||
assert rec.overall_state == AUTH_REQUIRED
|
||||
assert not ht.is_healthy("ag-test-1")
|
||||
|
||||
# Now update auth.json with newer mtime
|
||||
time.sleep(0.05)
|
||||
auth_file.write_text(json.dumps({"token": {"access_token": "new_refreshed_token"}}), encoding="utf-8")
|
||||
|
||||
# Calling is_healthy should detect the newer valid auth file and auto-recover
|
||||
assert ht.is_healthy("ag-test-1")
|
||||
rec = ht.get_or_create("ag-test-1")
|
||||
assert rec.overall_state == HEALTHY
|
||||
assert rec.last_error is None
|
||||
|
||||
|
||||
def test_clear_cooldown_full_reset(isolated_env):
|
||||
tmp_path, ht = isolated_env
|
||||
|
||||
ht.mark_auth_required("ag-1", reason="Invalid grant")
|
||||
rec = ht.get_or_create("ag-1")
|
||||
assert rec.overall_state == AUTH_REQUIRED
|
||||
assert rec.last_error == "Invalid grant"
|
||||
|
||||
# clear_cooldown should completely reset overall_state and last_error
|
||||
ht.clear_cooldown("ag-1")
|
||||
rec = ht.get_or_create("ag-1")
|
||||
assert rec.overall_state == HEALTHY
|
||||
assert rec.last_error is None
|
||||
for frec in rec.families.values():
|
||||
assert frec.state == HEALTHY
|
||||
assert frec.last_error is None
|
||||
assert frec.reason is None
|
||||
162
tests/test_model_validation_a23.py
Normal file
162
tests/test_model_validation_a23.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
"""Tests for model validation, suffix matching, and model discovery refresh."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("customtkinter")
|
||||
|
||||
from antigravity_provider.router.action_handler import ActionExecutor, do_set_model
|
||||
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||
from antigravity_provider.router.router_config import (
|
||||
RolePolicy,
|
||||
RouterConfig,
|
||||
RouterProfileConfig,
|
||||
save_router_config,
|
||||
)
|
||||
from antigravity_provider.router.ui.model_catalog import CachedModels, refresh_models_async
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_hub(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
# Reset ModelDiscoveryService singleton
|
||||
ModelDiscoveryService._instance = None
|
||||
|
||||
# Setup a router configuration
|
||||
cfg = RouterConfig(
|
||||
profiles={
|
||||
"ag-orch-1": RouterProfileConfig(
|
||||
profile_id="ag-orch-1",
|
||||
provider="antigravity",
|
||||
enabled=True,
|
||||
preferred_models=["google-antigravity/gemini-2.5-pro"],
|
||||
),
|
||||
"codex-1": RouterProfileConfig(
|
||||
profile_id="codex-1",
|
||||
provider="openai-codex",
|
||||
enabled=True,
|
||||
preferred_models=["gpt-4o"],
|
||||
),
|
||||
},
|
||||
roles={
|
||||
"orchestrator": RolePolicy(
|
||||
role_name="orchestrator",
|
||||
preferred_chain=["ag-orch-1"],
|
||||
),
|
||||
},
|
||||
default_role="orchestrator",
|
||||
)
|
||||
save_router_config(cfg)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_nonexistent_model_rejected_with_populated_cache(isolated_hub):
|
||||
"""Тест 1: Несуществующая модель отклоняется при наполненном кэше."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache["antigravity"] = {
|
||||
"models": ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-3.7-flash-high"],
|
||||
"discovered_at": 1000.0,
|
||||
}
|
||||
|
||||
ok, msg = do_set_model("ag-orch-1", "completely-fake-model-xyz")
|
||||
assert ok is False
|
||||
assert "отсутствует в списке обнаруженных моделей провайдера 'antigravity'" in msg
|
||||
|
||||
|
||||
def test_unknown_model_rejected_with_empty_cache(isolated_hub):
|
||||
"""Тест 2: При пустом кэше неизвестная модель отклоняется с внятной ошибкой."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache.clear()
|
||||
|
||||
with patch.object(service, "discover_models_sync", return_value=None):
|
||||
ok, msg = do_set_model("ag-orch-1", "unknown-unregistered-model")
|
||||
assert ok is False
|
||||
assert msg == "Кэш моделей для провайдера 'antigravity' пуст, а модель 'unknown-unregistered-model' не найдена в списке известных моделей."
|
||||
|
||||
|
||||
def test_base_name_without_effort_suffix_accepted(isolated_hub):
|
||||
"""Тест 3: Базовое имя без суффикса усилия (например gemini-3.7-flash) принимается, если в кэше есть gemini-3.7-flash-high."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache["antigravity"] = {
|
||||
"models": ["gemini-3.7-flash-high", "gemini-3.7-flash-medium"],
|
||||
"discovered_at": 1000.0,
|
||||
}
|
||||
|
||||
ok, msg = do_set_model("ag-orch-1", "gemini-3.7-flash")
|
||||
assert ok is True
|
||||
assert "успешно сохранена" in msg
|
||||
|
||||
|
||||
def test_canonical_model_accepted_even_with_empty_cache(isolated_hub):
|
||||
"""Каноническая модель из ModelRegistry принимается даже при пустом кэше discovery."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache.clear()
|
||||
|
||||
with patch.object(service, "discover_models_sync", return_value=None):
|
||||
ok, msg = do_set_model("ag-orch-1", "gemini-2.5-pro")
|
||||
assert ok is True
|
||||
assert "успешно сохранена" in msg
|
||||
|
||||
|
||||
def test_action_executor_refresh_models(isolated_hub):
|
||||
"""ActionExecutor.execute handles 'refresh_models' action."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with patch.object(service, "discover_models_sync", return_value=["test-model-1"]):
|
||||
res = ActionExecutor.execute("refresh_models", {"provider": "antigravity"})
|
||||
assert res.get("ok") is True
|
||||
assert res.get("data") == ["test-model-1"]
|
||||
|
||||
with patch.object(service, "refresh_all_async") as mock_refresh_all:
|
||||
res = ActionExecutor.execute("refresh_models", {})
|
||||
assert res.get("ok") is True
|
||||
mock_refresh_all.assert_called_once()
|
||||
|
||||
|
||||
def test_model_discovery_cache_retained_on_timeout_and_error(isolated_hub):
|
||||
"""Cache is not overwritten or cleared when model discovery probe times out or raises error."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache["antigravity"] = {
|
||||
"models": ["existing-gemini-model"],
|
||||
"discovered_at": 1000.0,
|
||||
}
|
||||
|
||||
# Simulate error in _probe_provider
|
||||
with patch.object(service, "_probe_provider", side_effect=RuntimeError("Network failure")):
|
||||
res = service.discover_models_sync("antigravity", timeout=1.0)
|
||||
assert res == ["existing-gemini-model"]
|
||||
assert service.get_models("antigravity") == ["existing-gemini-model"]
|
||||
|
||||
|
||||
def test_ui_model_catalog_refresh_models_async(isolated_hub):
|
||||
"""model_catalog.refresh_models_async dispatches and completes via service."""
|
||||
service = ModelDiscoveryService.get()
|
||||
with service._cache_lock:
|
||||
service._cache["antigravity"] = {
|
||||
"models": ["gemini-2.5-pro"],
|
||||
"discovered_at": 1000.0,
|
||||
}
|
||||
|
||||
completed: list[CachedModels] = []
|
||||
done_evt = threading.Event()
|
||||
|
||||
def _on_done(cm: CachedModels) -> None:
|
||||
completed.append(cm)
|
||||
done_evt.set()
|
||||
|
||||
with patch.object(service, "_probe_provider", return_value=["gemini-2.5-pro", "gemini-2.5-flash"]):
|
||||
ok = refresh_models_async("antigravity", _on_done)
|
||||
assert ok is True
|
||||
assert done_evt.wait(timeout=3.0) is True
|
||||
assert len(completed) == 1
|
||||
assert "gemini-2.5-pro" in completed[0].models
|
||||
Loading…
Reference in a new issue