feat(router): profile auth self-healing, honest model validation, and manual model refresh (A23)
This commit is contained in:
parent
9c6a4e8f6d
commit
1e1b81665b
9 changed files with 584 additions and 9 deletions
|
|
@ -139,6 +139,28 @@ def do_save_settings(settings: Dict[str, Any]) -> Tuple[bool, str]:
|
||||||
return True, "Настройки сохранены"
|
return True, "Настройки сохранены"
|
||||||
|
|
||||||
|
|
||||||
|
def _model_matches_discovered(model: str, discovered: list[str]) -> bool:
|
||||||
|
if model in discovered:
|
||||||
|
return True
|
||||||
|
model_short = model.split("/")[-1]
|
||||||
|
for d in discovered:
|
||||||
|
if d == model or d == model_short:
|
||||||
|
return True
|
||||||
|
d_short = d.split("/")[-1]
|
||||||
|
if d_short == model or d_short == model_short:
|
||||||
|
return True
|
||||||
|
if d.startswith(model + "-") or d.startswith(model + ":"):
|
||||||
|
return True
|
||||||
|
if d_short.startswith(model_short + "-") or d_short.startswith(model_short + ":"):
|
||||||
|
return True
|
||||||
|
import re
|
||||||
|
d_base = re.sub(r"-(high|medium|low|none|thought|thinking)(?:-(high|medium|low|none))?$", "", d_short)
|
||||||
|
model_base = re.sub(r"-(high|medium|low|none|thought|thinking)(?:-(high|medium|low|none))?$", "", model_short)
|
||||||
|
if d_base == model_short or d_base == model_base or d_short == model_base:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def do_set_model(profile_id: str, model: str, role_id: Optional[str] = None) -> Tuple[bool, str]:
|
def do_set_model(profile_id: str, model: str, role_id: Optional[str] = None) -> Tuple[bool, str]:
|
||||||
if not model or not str(model).strip() or str(model).strip() == "Список моделей ещё не получен":
|
if not model or not str(model).strip() or str(model).strip() == "Список моделей ещё не получен":
|
||||||
return False, "Не указана модель для установки"
|
return False, "Не указана модель для установки"
|
||||||
|
|
@ -160,11 +182,27 @@ def do_set_model(profile_id: str, model: str, role_id: Optional[str] = None) ->
|
||||||
from antigravity_provider.router.model_registry import ModelRegistry
|
from antigravity_provider.router.model_registry import ModelRegistry
|
||||||
|
|
||||||
discovered = ModelDiscoveryService.get().get_models(provider)
|
discovered = ModelDiscoveryService.get().get_models(provider)
|
||||||
|
if discovered is None:
|
||||||
|
try:
|
||||||
|
discovered = ModelDiscoveryService.get().discover_models_sync(provider, timeout=5.0)
|
||||||
|
except Exception:
|
||||||
|
discovered = None
|
||||||
|
|
||||||
canonical = [m.model_id for m in ModelRegistry.get().list_models(provider=provider)]
|
canonical = [m.model_id for m in ModelRegistry.get().list_models(provider=provider)]
|
||||||
canonical_short = [m.split("/")[-1] for m in canonical]
|
canonical_short = [m.split("/")[-1] for m in canonical]
|
||||||
|
|
||||||
if discovered is not None:
|
is_canonical = (
|
||||||
if model not in discovered and model not in canonical and model not in canonical_short:
|
model in canonical
|
||||||
|
or model in canonical_short
|
||||||
|
or _model_matches_discovered(model, canonical)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not discovered:
|
||||||
|
if not is_canonical:
|
||||||
|
return False, f"Кэш моделей для провайдера '{provider}' пуст, а модель '{model}' не найдена в списке известных моделей."
|
||||||
|
else:
|
||||||
|
matches_discovered = _model_matches_discovered(model, discovered)
|
||||||
|
if not matches_discovered and not is_canonical:
|
||||||
return False, f"Модель '{model}' отсутствует в списке обнаруженных моделей провайдера '{provider}'"
|
return False, f"Модель '{model}' отсутствует в списке обнаруженных моделей провайдера '{provider}'"
|
||||||
|
|
||||||
updated = load_router_config()
|
updated = load_router_config()
|
||||||
|
|
@ -257,6 +295,24 @@ class ActionExecutor:
|
||||||
else:
|
else:
|
||||||
HermesRefreshScheduler.get().trigger_refresh_account(prov, pid)
|
HermesRefreshScheduler.get().trigger_refresh_account(prov, pid)
|
||||||
return {'ok': True, 'message': 'Успешно'}
|
return {'ok': True, 'message': 'Успешно'}
|
||||||
|
|
||||||
|
elif action == 'refresh_models':
|
||||||
|
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
if prov:
|
||||||
|
if async_runner:
|
||||||
|
async_runner(lambda: service.discover_models_sync(prov), 'RefreshModels')
|
||||||
|
return {'ok': True, 'message': 'запущено'}
|
||||||
|
else:
|
||||||
|
res = service.discover_models_sync(prov)
|
||||||
|
return {'ok': True, 'message': 'Успешно', 'data': res}
|
||||||
|
else:
|
||||||
|
if async_runner:
|
||||||
|
async_runner(lambda: service.refresh_all_async(), 'RefreshAllModels')
|
||||||
|
return {'ok': True, 'message': 'запущено'}
|
||||||
|
else:
|
||||||
|
service.refresh_all_async()
|
||||||
|
return {'ok': True, 'message': 'запущено'}
|
||||||
|
|
||||||
elif action == 'save_settings':
|
elif action == 'save_settings':
|
||||||
ok, msg = do_save_settings(data)
|
ok, msg = do_save_settings(data)
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,7 @@ class ProfileHealthRecord:
|
||||||
last_used: Optional[float] = None
|
last_used: Optional[float] = None
|
||||||
last_success: Optional[float] = None
|
last_success: Optional[float] = None
|
||||||
last_error: Optional[str] = None
|
last_error: Optional[str] = None
|
||||||
|
auth_error_at: Optional[float] = None
|
||||||
simulated: bool = False
|
simulated: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -76,12 +77,18 @@ class _FileLock:
|
||||||
self._fd = os.open(str(self.lock_path), os.O_CREAT | os.O_RDWR)
|
self._fd = os.open(str(self.lock_path), os.O_CREAT | os.O_RDWR)
|
||||||
if os.name == "nt":
|
if os.name == "nt":
|
||||||
import msvcrt
|
import msvcrt
|
||||||
msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1)
|
msvcrt.locking(self._fd, msvcrt.LK_LOCK, 1)
|
||||||
else:
|
else:
|
||||||
import fcntl
|
import fcntl
|
||||||
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
fcntl.flock(self._fd, fcntl.LOCK_EX)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
if self._fd is not None:
|
||||||
|
try:
|
||||||
|
os.close(self._fd)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._fd = None
|
||||||
|
raise
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
|
@ -118,6 +125,99 @@ class HealthTracker:
|
||||||
self._profiles: dict[str, ProfileHealthRecord] = {}
|
self._profiles: dict[str, ProfileHealthRecord] = {}
|
||||||
self._load_state()
|
self._load_state()
|
||||||
|
|
||||||
|
try:
|
||||||
|
from antigravity_provider.router.event_bus import (
|
||||||
|
EVENT_ACCOUNT_ADDED,
|
||||||
|
EVENT_ACCOUNT_AUTH_CHANGED,
|
||||||
|
EventBus,
|
||||||
|
)
|
||||||
|
EventBus.get().subscribe(EVENT_ACCOUNT_ADDED, self._on_account_event)
|
||||||
|
EventBus.get().subscribe(EVENT_ACCOUNT_AUTH_CHANGED, self._on_account_event)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _on_account_event(self, _event_name: str, payload: Any) -> None:
|
||||||
|
"""Handle account lifecycle events by recovering profile from AUTH_REQUIRED state."""
|
||||||
|
profile_id = None
|
||||||
|
if isinstance(payload, dict):
|
||||||
|
profile_id = payload.get("profile_id")
|
||||||
|
elif hasattr(payload, "profile_id"):
|
||||||
|
profile_id = getattr(payload, "profile_id")
|
||||||
|
if not profile_id:
|
||||||
|
return
|
||||||
|
profile_id = str(profile_id)
|
||||||
|
with self._lock:
|
||||||
|
if profile_id in self._profiles:
|
||||||
|
rec = self._profiles[profile_id]
|
||||||
|
rec.overall_state = HEALTHY
|
||||||
|
rec.last_error = None
|
||||||
|
rec.auth_error_at = None
|
||||||
|
for frec in rec.families.values():
|
||||||
|
if frec.state == AUTH_REQUIRED:
|
||||||
|
frec.state = HEALTHY
|
||||||
|
frec.reset_at = None
|
||||||
|
frec.reason = None
|
||||||
|
frec.last_error = None
|
||||||
|
self._save_state()
|
||||||
|
|
||||||
|
def _find_profile_auth_files(self, profile_id: str) -> list[Path]:
|
||||||
|
"""Locate credentials and auth files for the specified profile."""
|
||||||
|
files: list[Path] = []
|
||||||
|
try:
|
||||||
|
pdir = paths.get_profile_dir(profile_id)
|
||||||
|
for cand in (pdir / "auth.json", pdir / ".gemini" / "oauth_creds.json", pdir / "oauth_creds.json"):
|
||||||
|
if cand.is_file():
|
||||||
|
files.append(cand)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
hermes_home = paths.get_hermes_home()
|
||||||
|
if hermes_home.is_dir():
|
||||||
|
for pdir in hermes_home.glob(f"*_profiles/{profile_id}"):
|
||||||
|
if pdir.is_dir():
|
||||||
|
for cand in (pdir / "auth.json", pdir / ".gemini" / "oauth_creds.json", pdir / "oauth_creds.json"):
|
||||||
|
if cand.is_file() and cand not in files:
|
||||||
|
files.append(cand)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return files
|
||||||
|
|
||||||
|
def _check_and_recover_auth(self, record: ProfileHealthRecord) -> bool:
|
||||||
|
"""Check if auth files exist, are valid, and were modified after an auth failure."""
|
||||||
|
auth_files = self._find_profile_auth_files(record.profile_id)
|
||||||
|
if not auth_files:
|
||||||
|
return False
|
||||||
|
|
||||||
|
recovered = False
|
||||||
|
for f in auth_files:
|
||||||
|
try:
|
||||||
|
stat = f.stat()
|
||||||
|
if stat.st_size == 0:
|
||||||
|
continue
|
||||||
|
if record.auth_error_at is not None and stat.st_mtime <= record.auth_error_at:
|
||||||
|
continue
|
||||||
|
content = json.loads(f.read_text(encoding="utf-8"))
|
||||||
|
if isinstance(content, dict) and len(content) > 0:
|
||||||
|
recovered = True
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if recovered:
|
||||||
|
record.overall_state = HEALTHY
|
||||||
|
record.last_error = None
|
||||||
|
record.auth_error_at = None
|
||||||
|
for frec in record.families.values():
|
||||||
|
if frec.state == AUTH_REQUIRED:
|
||||||
|
frec.state = HEALTHY
|
||||||
|
frec.reset_at = None
|
||||||
|
frec.reason = None
|
||||||
|
frec.last_error = None
|
||||||
|
self._save_state()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _load_state(self) -> None:
|
def _load_state(self) -> None:
|
||||||
if not self.state_file.is_file():
|
if not self.state_file.is_file():
|
||||||
return
|
return
|
||||||
|
|
@ -130,6 +230,7 @@ class HealthTracker:
|
||||||
last_used=pdata.get("last_used"),
|
last_used=pdata.get("last_used"),
|
||||||
last_success=pdata.get("last_success"),
|
last_success=pdata.get("last_success"),
|
||||||
last_error=pdata.get("last_error"),
|
last_error=pdata.get("last_error"),
|
||||||
|
auth_error_at=pdata.get("auth_error_at"),
|
||||||
simulated=pdata.get("simulated", False),
|
simulated=pdata.get("simulated", False),
|
||||||
)
|
)
|
||||||
for fname, fdata in pdata.get("families", {}).items():
|
for fname, fdata in pdata.get("families", {}).items():
|
||||||
|
|
@ -158,6 +259,7 @@ class HealthTracker:
|
||||||
"last_used": precord.last_used,
|
"last_used": precord.last_used,
|
||||||
"last_success": precord.last_success,
|
"last_success": precord.last_success,
|
||||||
"last_error": precord.last_error,
|
"last_error": precord.last_error,
|
||||||
|
"auth_error_at": precord.auth_error_at,
|
||||||
"simulated": precord.simulated,
|
"simulated": precord.simulated,
|
||||||
"families": {},
|
"families": {},
|
||||||
}
|
}
|
||||||
|
|
@ -211,6 +313,9 @@ class HealthTracker:
|
||||||
if record.overall_state == DISABLED:
|
if record.overall_state == DISABLED:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if record.overall_state == AUTH_REQUIRED:
|
||||||
|
self._check_and_recover_auth(record)
|
||||||
|
|
||||||
# Check profile-level default family
|
# Check profile-level default family
|
||||||
if "default" in record.families:
|
if "default" in record.families:
|
||||||
def_rec = record.families["default"]
|
def_rec = record.families["default"]
|
||||||
|
|
@ -257,6 +362,7 @@ class HealthTracker:
|
||||||
record.last_used = now
|
record.last_used = now
|
||||||
record.last_success = now
|
record.last_success = now
|
||||||
record.overall_state = HEALTHY
|
record.overall_state = HEALTHY
|
||||||
|
record.auth_error_at = None
|
||||||
record.simulated = False
|
record.simulated = False
|
||||||
|
|
||||||
family = extract_model_family(model_name)
|
family = extract_model_family(model_name)
|
||||||
|
|
@ -378,6 +484,7 @@ class HealthTracker:
|
||||||
record = self.get_or_create(profile_id)
|
record = self.get_or_create(profile_id)
|
||||||
record.overall_state = AUTH_REQUIRED
|
record.overall_state = AUTH_REQUIRED
|
||||||
record.last_error = reason
|
record.last_error = reason
|
||||||
|
record.auth_error_at = time.time()
|
||||||
self._save_state()
|
self._save_state()
|
||||||
|
|
||||||
def clear_cooldown(self, profile_id: Optional[str] = None, model_name: Optional[str] = None) -> None:
|
def clear_cooldown(self, profile_id: Optional[str] = None, model_name: Optional[str] = None) -> None:
|
||||||
|
|
@ -385,10 +492,14 @@ class HealthTracker:
|
||||||
if profile_id is None:
|
if profile_id is None:
|
||||||
for rec in self._profiles.values():
|
for rec in self._profiles.values():
|
||||||
rec.overall_state = HEALTHY
|
rec.overall_state = HEALTHY
|
||||||
|
rec.last_error = None
|
||||||
|
rec.auth_error_at = None
|
||||||
rec.simulated = False
|
rec.simulated = False
|
||||||
for frec in rec.families.values():
|
for frec in rec.families.values():
|
||||||
frec.state = HEALTHY
|
frec.state = HEALTHY
|
||||||
frec.reset_at = None
|
frec.reset_at = None
|
||||||
|
frec.reason = None
|
||||||
|
frec.last_error = None
|
||||||
frec.simulated = False
|
frec.simulated = False
|
||||||
self._save_state()
|
self._save_state()
|
||||||
return
|
return
|
||||||
|
|
@ -397,16 +508,22 @@ class HealthTracker:
|
||||||
return
|
return
|
||||||
record = self._profiles[profile_id]
|
record = self._profiles[profile_id]
|
||||||
record.overall_state = HEALTHY
|
record.overall_state = HEALTHY
|
||||||
|
record.last_error = None
|
||||||
|
record.auth_error_at = None
|
||||||
record.simulated = False
|
record.simulated = False
|
||||||
if model_name:
|
if model_name:
|
||||||
family = extract_model_family(model_name)
|
family = extract_model_family(model_name)
|
||||||
if family in record.families:
|
if family in record.families:
|
||||||
record.families[family].state = HEALTHY
|
record.families[family].state = HEALTHY
|
||||||
record.families[family].reset_at = None
|
record.families[family].reset_at = None
|
||||||
|
record.families[family].reason = None
|
||||||
|
record.families[family].last_error = None
|
||||||
record.families[family].simulated = False
|
record.families[family].simulated = False
|
||||||
else:
|
else:
|
||||||
for frec in record.families.values():
|
for frec in record.families.values():
|
||||||
frec.state = HEALTHY
|
frec.state = HEALTHY
|
||||||
frec.reset_at = None
|
frec.reset_at = None
|
||||||
|
frec.reason = None
|
||||||
|
frec.last_error = None
|
||||||
frec.simulated = False
|
frec.simulated = False
|
||||||
self._save_state()
|
self._save_state()
|
||||||
|
|
|
||||||
|
|
@ -198,7 +198,7 @@ class ModelDiscoveryService:
|
||||||
return list(entry["models"]) if entry and "models" in entry else None
|
return list(entry["models"]) if entry and "models" in entry else None
|
||||||
|
|
||||||
models = result_holder[0]
|
models = result_holder[0]
|
||||||
if models is not None:
|
if models:
|
||||||
with self._cache_lock:
|
with self._cache_lock:
|
||||||
self._cache[provider.lower()] = {
|
self._cache[provider.lower()] = {
|
||||||
"models": models,
|
"models": models,
|
||||||
|
|
@ -208,7 +208,7 @@ class ModelDiscoveryService:
|
||||||
logger.info("Discovered %d models for provider '%s': %s", len(models), provider, models)
|
logger.info("Discovered %d models for provider '%s': %s", len(models), provider, models)
|
||||||
return models
|
return models
|
||||||
|
|
||||||
# If probe returned None, retain existing cache if any
|
# If probe returned None or empty, retain existing cache if any
|
||||||
with self._cache_lock:
|
with self._cache_lock:
|
||||||
entry = self._cache.get(provider.lower())
|
entry = self._cache.get(provider.lower())
|
||||||
return list(entry["models"]) if entry and "models" in entry else None
|
return list(entry["models"]) if entry and "models" in entry else None
|
||||||
|
|
|
||||||
|
|
@ -216,6 +216,51 @@ class ModelRegistry:
|
||||||
quota_bucket="antigravity.gemini",
|
quota_bucket="antigravity.gemini",
|
||||||
quality_tier=3,
|
quality_tier=3,
|
||||||
),
|
),
|
||||||
|
ModelDescriptor(
|
||||||
|
model_id="google-antigravity/gemini-3.1-pro",
|
||||||
|
display_name="Gemini 3.1 Pro",
|
||||||
|
provider="antigravity",
|
||||||
|
family="gemini",
|
||||||
|
capabilities=["coding", "reasoning", "tools", "structured_output", "long_context", "planning", "security_analysis"],
|
||||||
|
context_window=1000000,
|
||||||
|
supports_tools=True,
|
||||||
|
supports_reasoning=True,
|
||||||
|
latency_class="medium",
|
||||||
|
cost_input_per_m=1.25,
|
||||||
|
cost_output_per_m=5.0,
|
||||||
|
quota_bucket="antigravity.gemini",
|
||||||
|
quality_tier=5,
|
||||||
|
),
|
||||||
|
ModelDescriptor(
|
||||||
|
model_id="google-antigravity/gemini-3.7-flash",
|
||||||
|
display_name="Gemini 3.7 Flash",
|
||||||
|
provider="antigravity",
|
||||||
|
family="gemini",
|
||||||
|
capabilities=["coding", "reasoning", "tools", "structured_output", "classification", "routing", "long_context"],
|
||||||
|
context_window=1000000,
|
||||||
|
supports_tools=True,
|
||||||
|
supports_reasoning=True,
|
||||||
|
latency_class="ultra_low",
|
||||||
|
cost_input_per_m=0.15,
|
||||||
|
cost_output_per_m=0.6,
|
||||||
|
quota_bucket="antigravity.gemini",
|
||||||
|
quality_tier=4,
|
||||||
|
),
|
||||||
|
ModelDescriptor(
|
||||||
|
model_id="google-antigravity/gemini-3.7-pro",
|
||||||
|
display_name="Gemini 3.7 Pro",
|
||||||
|
provider="antigravity",
|
||||||
|
family="gemini",
|
||||||
|
capabilities=["coding", "reasoning", "tools", "structured_output", "long_context", "planning", "security_analysis"],
|
||||||
|
context_window=1000000,
|
||||||
|
supports_tools=True,
|
||||||
|
supports_reasoning=True,
|
||||||
|
latency_class="medium",
|
||||||
|
cost_input_per_m=1.25,
|
||||||
|
cost_output_per_m=5.0,
|
||||||
|
quota_bucket="antigravity.gemini",
|
||||||
|
quality_tier=5,
|
||||||
|
),
|
||||||
# Google Antigravity (Claude family inside AGY)
|
# Google Antigravity (Claude family inside AGY)
|
||||||
ModelDescriptor(
|
ModelDescriptor(
|
||||||
model_id="google-antigravity/claude-3-7-sonnet",
|
model_id="google-antigravity/claude-3-7-sonnet",
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,13 @@ def _service() -> Any:
|
||||||
try:
|
try:
|
||||||
from antigravity_provider.router.model_discovery import ModelDiscoveryService
|
from antigravity_provider.router.model_discovery import ModelDiscoveryService
|
||||||
|
|
||||||
|
getter = getattr(ModelDiscoveryService, "get", None)
|
||||||
|
return getter() if callable(getter) else ModelDiscoveryService()
|
||||||
|
except (ImportError, AttributeError, TypeError):
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||||
|
|
||||||
getter = getattr(ModelDiscoveryService, "get", None)
|
getter = getattr(ModelDiscoveryService, "get", None)
|
||||||
return getter() if callable(getter) else ModelDiscoveryService()
|
return getter() if callable(getter) else ModelDiscoveryService()
|
||||||
except (ImportError, AttributeError, TypeError):
|
except (ImportError, AttributeError, TypeError):
|
||||||
|
|
@ -80,7 +87,7 @@ def refresh_models_async(provider: str, on_complete: Callable[[CachedModels], No
|
||||||
if service is None:
|
if service is None:
|
||||||
on_complete(get_cached_models(provider))
|
on_complete(get_cached_models(provider))
|
||||||
return False
|
return False
|
||||||
for name in ("refresh_provider_async", "refresh_async", "discover_async"):
|
for name in ("refresh_models_async", "refresh_models", "refresh_provider_async", "refresh_async", "discover_async"):
|
||||||
method = getattr(service, name, None)
|
method = getattr(service, name, None)
|
||||||
if not callable(method):
|
if not callable(method):
|
||||||
continue
|
continue
|
||||||
|
|
|
||||||
|
|
@ -703,6 +703,16 @@ class UnifiedHealthService:
|
||||||
for p in profs:
|
for p in profs:
|
||||||
for m in p.preferred_models:
|
for m in p.preferred_models:
|
||||||
models_set.add(m)
|
models_set.add(m)
|
||||||
|
|
||||||
|
# Add discovered models from ModelDiscoveryService cache
|
||||||
|
try:
|
||||||
|
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||||
|
cached = ModelDiscoveryService.get().get_cached(prov_id)
|
||||||
|
if cached and cached.get("has_cache") and cached.get("models"):
|
||||||
|
for m in cached["models"]:
|
||||||
|
models_set.add(m)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
summaries.append(ProviderSummary(
|
summaries.append(ProviderSummary(
|
||||||
provider_id=prov_id,
|
provider_id=prov_id,
|
||||||
|
|
|
||||||
|
|
@ -1375,7 +1375,7 @@ async function handleSaveRoleModel(roleId, profileId) {
|
||||||
|
|
||||||
async function handleRefreshProviderModels(providerId, profileId = null) {
|
async function handleRefreshProviderModels(providerId, profileId = null) {
|
||||||
showToast(`Запрос списка моделей для ${providerId}...`, 'info');
|
showToast(`Запрос списка моделей для ${providerId}...`, 'info');
|
||||||
const res = await executeAction('refresh_data', { provider: providerId });
|
const res = await executeAction('refresh_models', { provider: providerId });
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
showToast('Запрос обновления моделей отправлен', 'success');
|
showToast('Запрос обновления моделей отправлен', 'success');
|
||||||
if (profileId) {
|
if (profileId) {
|
||||||
|
|
|
||||||
178
tests/test_health_auth_recovery.py
Normal file
178
tests/test_health_auth_recovery.py
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
"""Tests for health tracker and router profile self-recovery from AUTH_REQUIRED state."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from antigravity_provider.router.adapters.antigravity_adapter import AntigravityAdapter
|
||||||
|
from antigravity_provider.router.health_tracker import (
|
||||||
|
AUTH_REQUIRED,
|
||||||
|
HEALTHY,
|
||||||
|
HealthTracker,
|
||||||
|
)
|
||||||
|
from antigravity_provider.router.profile_manager import ProfileAuthManager
|
||||||
|
from antigravity_provider.router.router_config import (
|
||||||
|
RolePolicy,
|
||||||
|
RouterConfig,
|
||||||
|
RouterProfileConfig,
|
||||||
|
)
|
||||||
|
from antigravity_provider.router.router_engine import RouterEngine
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def isolated_env(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||||
|
state_file = tmp_path / "router_state.json"
|
||||||
|
ht = HealthTracker(state_file=state_file)
|
||||||
|
return tmp_path, ht
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_required_skipped_in_routing(isolated_env):
|
||||||
|
tmp_path, ht = isolated_env
|
||||||
|
|
||||||
|
# Configure 2 profiles for role 'orchestrator'
|
||||||
|
cfg = RouterConfig(
|
||||||
|
profiles={
|
||||||
|
"ag-orch-1": RouterProfileConfig(
|
||||||
|
profile_id="ag-orch-1",
|
||||||
|
provider="antigravity",
|
||||||
|
enabled=True,
|
||||||
|
),
|
||||||
|
"ag-orch-2": RouterProfileConfig(
|
||||||
|
profile_id="ag-orch-2",
|
||||||
|
provider="antigravity",
|
||||||
|
enabled=True,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
roles={
|
||||||
|
"orchestrator": RolePolicy(
|
||||||
|
role_name="orchestrator",
|
||||||
|
preferred_chain=["ag-orch-1", "ag-orch-2"],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
default_role="orchestrator",
|
||||||
|
)
|
||||||
|
|
||||||
|
engine = RouterEngine(config=cfg, health=ht)
|
||||||
|
|
||||||
|
def _mock_invoke(*args, **kwargs):
|
||||||
|
return {"id": "res-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]}
|
||||||
|
|
||||||
|
# Initial state: ag-orch-1 is healthy and selected
|
||||||
|
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||||
|
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||||
|
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-1"
|
||||||
|
|
||||||
|
# Mark ag-orch-1 as AUTH_REQUIRED
|
||||||
|
ht.mark_auth_required("ag-orch-1", reason="401 Unauthorized token expired")
|
||||||
|
assert not ht.is_healthy("ag-orch-1")
|
||||||
|
|
||||||
|
# Routing should skip ag-orch-1 and route to ag-orch-2
|
||||||
|
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||||
|
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||||
|
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_profile_auth_auto_recovery(isolated_env):
|
||||||
|
tmp_path, ht = isolated_env
|
||||||
|
|
||||||
|
cfg = RouterConfig(
|
||||||
|
profiles={
|
||||||
|
"ag-orch-1": RouterProfileConfig(
|
||||||
|
profile_id="ag-orch-1",
|
||||||
|
provider="antigravity",
|
||||||
|
enabled=True,
|
||||||
|
),
|
||||||
|
"ag-orch-2": RouterProfileConfig(
|
||||||
|
profile_id="ag-orch-2",
|
||||||
|
provider="antigravity",
|
||||||
|
enabled=True,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
roles={
|
||||||
|
"orchestrator": RolePolicy(
|
||||||
|
role_name="orchestrator",
|
||||||
|
preferred_chain=["ag-orch-1", "ag-orch-2"],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
default_role="orchestrator",
|
||||||
|
)
|
||||||
|
engine = RouterEngine(config=cfg, health=ht)
|
||||||
|
|
||||||
|
def _mock_invoke(*args, **kwargs):
|
||||||
|
return {"id": "res-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]}
|
||||||
|
|
||||||
|
# Mark ag-orch-1 as AUTH_REQUIRED
|
||||||
|
ht.mark_auth_required("ag-orch-1", reason="Auth token rejected")
|
||||||
|
rec = ht.get_or_create("ag-orch-1")
|
||||||
|
assert rec.overall_state == AUTH_REQUIRED
|
||||||
|
assert rec.last_error == "Auth token rejected"
|
||||||
|
assert not ht.is_healthy("ag-orch-1")
|
||||||
|
|
||||||
|
# Saving credentials via ProfileAuthManager publishes event and auto-recovers health
|
||||||
|
ProfileAuthManager.save_profile_auth(
|
||||||
|
"antigravity",
|
||||||
|
"ag-orch-1",
|
||||||
|
{"token": {"access_token": "valid_tok_123", "refresh_token": "ref_123"}},
|
||||||
|
)
|
||||||
|
|
||||||
|
# State is automatically restored to HEALTHY without manual clear_cooldown
|
||||||
|
rec = ht.get_or_create("ag-orch-1")
|
||||||
|
assert rec.overall_state == HEALTHY
|
||||||
|
assert rec.last_error is None
|
||||||
|
assert ht.is_healthy("ag-orch-1")
|
||||||
|
|
||||||
|
# Router now routes back to primary profile ag-orch-1
|
||||||
|
with patch.object(AntigravityAdapter, "invoke", side_effect=_mock_invoke):
|
||||||
|
res = engine.route_request({"prompt": "hi"}, role="orchestrator")
|
||||||
|
assert res.get("router_metadata", {}).get("profile_id") == "ag-orch-1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_file_mtime_recovery_in_is_healthy(isolated_env):
|
||||||
|
tmp_path, ht = isolated_env
|
||||||
|
|
||||||
|
# Setup profile directory and initial auth file
|
||||||
|
pdir = tmp_path / "agy_profiles" / "ag-test-1"
|
||||||
|
pdir.mkdir(parents=True, exist_ok=True)
|
||||||
|
auth_file = pdir / "auth.json"
|
||||||
|
auth_file.write_text(json.dumps({"token": "old"}), encoding="utf-8")
|
||||||
|
time.sleep(0.05)
|
||||||
|
|
||||||
|
# Mark AUTH_REQUIRED at a timestamp
|
||||||
|
ht.mark_auth_required("ag-test-1", reason="OAuth error")
|
||||||
|
rec = ht.get_or_create("ag-test-1")
|
||||||
|
assert rec.overall_state == AUTH_REQUIRED
|
||||||
|
assert not ht.is_healthy("ag-test-1")
|
||||||
|
|
||||||
|
# Now update auth.json with newer mtime
|
||||||
|
time.sleep(0.05)
|
||||||
|
auth_file.write_text(json.dumps({"token": {"access_token": "new_refreshed_token"}}), encoding="utf-8")
|
||||||
|
|
||||||
|
# Calling is_healthy should detect the newer valid auth file and auto-recover
|
||||||
|
assert ht.is_healthy("ag-test-1")
|
||||||
|
rec = ht.get_or_create("ag-test-1")
|
||||||
|
assert rec.overall_state == HEALTHY
|
||||||
|
assert rec.last_error is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_cooldown_full_reset(isolated_env):
|
||||||
|
tmp_path, ht = isolated_env
|
||||||
|
|
||||||
|
ht.mark_auth_required("ag-1", reason="Invalid grant")
|
||||||
|
rec = ht.get_or_create("ag-1")
|
||||||
|
assert rec.overall_state == AUTH_REQUIRED
|
||||||
|
assert rec.last_error == "Invalid grant"
|
||||||
|
|
||||||
|
# clear_cooldown should completely reset overall_state and last_error
|
||||||
|
ht.clear_cooldown("ag-1")
|
||||||
|
rec = ht.get_or_create("ag-1")
|
||||||
|
assert rec.overall_state == HEALTHY
|
||||||
|
assert rec.last_error is None
|
||||||
|
for frec in rec.families.values():
|
||||||
|
assert frec.state == HEALTHY
|
||||||
|
assert frec.last_error is None
|
||||||
|
assert frec.reason is None
|
||||||
162
tests/test_model_validation_a23.py
Normal file
162
tests/test_model_validation_a23.py
Normal file
|
|
@ -0,0 +1,162 @@
|
||||||
|
"""Tests for model validation, suffix matching, and model discovery refresh."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
pytest.importorskip("customtkinter")
|
||||||
|
|
||||||
|
from antigravity_provider.router.action_handler import ActionExecutor, do_set_model
|
||||||
|
from antigravity_provider.router.model_discovery_service import ModelDiscoveryService
|
||||||
|
from antigravity_provider.router.router_config import (
|
||||||
|
RolePolicy,
|
||||||
|
RouterConfig,
|
||||||
|
RouterProfileConfig,
|
||||||
|
save_router_config,
|
||||||
|
)
|
||||||
|
from antigravity_provider.router.ui.model_catalog import CachedModels, refresh_models_async
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def isolated_hub(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||||
|
|
||||||
|
# Reset ModelDiscoveryService singleton
|
||||||
|
ModelDiscoveryService._instance = None
|
||||||
|
|
||||||
|
# Setup a router configuration
|
||||||
|
cfg = RouterConfig(
|
||||||
|
profiles={
|
||||||
|
"ag-orch-1": RouterProfileConfig(
|
||||||
|
profile_id="ag-orch-1",
|
||||||
|
provider="antigravity",
|
||||||
|
enabled=True,
|
||||||
|
preferred_models=["google-antigravity/gemini-2.5-pro"],
|
||||||
|
),
|
||||||
|
"codex-1": RouterProfileConfig(
|
||||||
|
profile_id="codex-1",
|
||||||
|
provider="openai-codex",
|
||||||
|
enabled=True,
|
||||||
|
preferred_models=["gpt-4o"],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
roles={
|
||||||
|
"orchestrator": RolePolicy(
|
||||||
|
role_name="orchestrator",
|
||||||
|
preferred_chain=["ag-orch-1"],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
default_role="orchestrator",
|
||||||
|
)
|
||||||
|
save_router_config(cfg)
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonexistent_model_rejected_with_populated_cache(isolated_hub):
|
||||||
|
"""Тест 1: Несуществующая модель отклоняется при наполненном кэше."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache["antigravity"] = {
|
||||||
|
"models": ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-3.7-flash-high"],
|
||||||
|
"discovered_at": 1000.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, msg = do_set_model("ag-orch-1", "completely-fake-model-xyz")
|
||||||
|
assert ok is False
|
||||||
|
assert "отсутствует в списке обнаруженных моделей провайдера 'antigravity'" in msg
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_model_rejected_with_empty_cache(isolated_hub):
|
||||||
|
"""Тест 2: При пустом кэше неизвестная модель отклоняется с внятной ошибкой."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache.clear()
|
||||||
|
|
||||||
|
with patch.object(service, "discover_models_sync", return_value=None):
|
||||||
|
ok, msg = do_set_model("ag-orch-1", "unknown-unregistered-model")
|
||||||
|
assert ok is False
|
||||||
|
assert msg == "Кэш моделей для провайдера 'antigravity' пуст, а модель 'unknown-unregistered-model' не найдена в списке известных моделей."
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_name_without_effort_suffix_accepted(isolated_hub):
|
||||||
|
"""Тест 3: Базовое имя без суффикса усилия (например gemini-3.7-flash) принимается, если в кэше есть gemini-3.7-flash-high."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache["antigravity"] = {
|
||||||
|
"models": ["gemini-3.7-flash-high", "gemini-3.7-flash-medium"],
|
||||||
|
"discovered_at": 1000.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, msg = do_set_model("ag-orch-1", "gemini-3.7-flash")
|
||||||
|
assert ok is True
|
||||||
|
assert "успешно сохранена" in msg
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_model_accepted_even_with_empty_cache(isolated_hub):
|
||||||
|
"""Каноническая модель из ModelRegistry принимается даже при пустом кэше discovery."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache.clear()
|
||||||
|
|
||||||
|
with patch.object(service, "discover_models_sync", return_value=None):
|
||||||
|
ok, msg = do_set_model("ag-orch-1", "gemini-2.5-pro")
|
||||||
|
assert ok is True
|
||||||
|
assert "успешно сохранена" in msg
|
||||||
|
|
||||||
|
|
||||||
|
def test_action_executor_refresh_models(isolated_hub):
|
||||||
|
"""ActionExecutor.execute handles 'refresh_models' action."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with patch.object(service, "discover_models_sync", return_value=["test-model-1"]):
|
||||||
|
res = ActionExecutor.execute("refresh_models", {"provider": "antigravity"})
|
||||||
|
assert res.get("ok") is True
|
||||||
|
assert res.get("data") == ["test-model-1"]
|
||||||
|
|
||||||
|
with patch.object(service, "refresh_all_async") as mock_refresh_all:
|
||||||
|
res = ActionExecutor.execute("refresh_models", {})
|
||||||
|
assert res.get("ok") is True
|
||||||
|
mock_refresh_all.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_discovery_cache_retained_on_timeout_and_error(isolated_hub):
|
||||||
|
"""Cache is not overwritten or cleared when model discovery probe times out or raises error."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache["antigravity"] = {
|
||||||
|
"models": ["existing-gemini-model"],
|
||||||
|
"discovered_at": 1000.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Simulate error in _probe_provider
|
||||||
|
with patch.object(service, "_probe_provider", side_effect=RuntimeError("Network failure")):
|
||||||
|
res = service.discover_models_sync("antigravity", timeout=1.0)
|
||||||
|
assert res == ["existing-gemini-model"]
|
||||||
|
assert service.get_models("antigravity") == ["existing-gemini-model"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_ui_model_catalog_refresh_models_async(isolated_hub):
|
||||||
|
"""model_catalog.refresh_models_async dispatches and completes via service."""
|
||||||
|
service = ModelDiscoveryService.get()
|
||||||
|
with service._cache_lock:
|
||||||
|
service._cache["antigravity"] = {
|
||||||
|
"models": ["gemini-2.5-pro"],
|
||||||
|
"discovered_at": 1000.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
completed: list[CachedModels] = []
|
||||||
|
done_evt = threading.Event()
|
||||||
|
|
||||||
|
def _on_done(cm: CachedModels) -> None:
|
||||||
|
completed.append(cm)
|
||||||
|
done_evt.set()
|
||||||
|
|
||||||
|
with patch.object(service, "_probe_provider", return_value=["gemini-2.5-pro", "gemini-2.5-flash"]):
|
||||||
|
ok = refresh_models_async("antigravity", _on_done)
|
||||||
|
assert ok is True
|
||||||
|
assert done_evt.wait(timeout=3.0) is True
|
||||||
|
assert len(completed) == 1
|
||||||
|
assert "gemini-2.5-pro" in completed[0].models
|
||||||
Loading…
Reference in a new issue