8c6bc7a7ed
- Measure Qwen3-Coder-30B-A3B, Qwen2.5-Coder-32B, and Tiel-Coder-35B-A3B at 64k and 32k context - Verify long-context degradation profile and MoE attention scaling - Add benchmark suite and results to BENCHMARK_MOE_CANDIDATES.md and benchmark_moe_results.json
417 lines
21 KiB
Python
417 lines
21 KiB
Python
"""Automated Evaluation Suite for Local LLM Benchmarking on Hermes Hub Codebase Tasks."""
|
||
from __future__ import annotations
|
||
|
||
import ast
|
||
import re
|
||
from dataclasses import dataclass
|
||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||
|
||
|
||
@dataclass
|
||
class BenchmarkTask:
|
||
task_id: str
|
||
title: str
|
||
category: str
|
||
prompt: str
|
||
expected_function_name: str
|
||
test_function: Callable[[Any], Tuple[bool, str]]
|
||
is_long_context: bool = False
|
||
context_data: Optional[str] = None
|
||
|
||
|
||
def _clean_code(text: str) -> str:
|
||
"""Extract python code from model output, stripping markdown, thoughts, and conversational fluff."""
|
||
if not text:
|
||
return ""
|
||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
|
||
code_block = re.search(r"```(?:python|py)?\s*(.*?)\s*```", text, re.DOTALL)
|
||
if code_block:
|
||
return code_block.group(1).strip()
|
||
return text.strip()
|
||
|
||
|
||
def _compile_and_get(code_str: str, target_symbol: str) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Compile Python code and retrieve the target symbol in a clean namespace."""
|
||
clean = _clean_code(code_str)
|
||
try:
|
||
ast.parse(clean)
|
||
except SyntaxError as e:
|
||
return None, f"SyntaxError: {e}"
|
||
|
||
namespace: Dict[str, Any] = {}
|
||
try:
|
||
exec(clean, namespace)
|
||
except Exception as e:
|
||
return None, f"RuntimeError on load: {type(e).__name__}: {e}"
|
||
|
||
if target_symbol not in namespace:
|
||
return None, f"Symbol '{target_symbol}' not found in generated code"
|
||
|
||
return namespace[target_symbol], None
|
||
|
||
|
||
# Task 1
|
||
def test_t1(fn) -> Tuple[bool, str]:
|
||
cases = [
|
||
("gemini-3.7-flash", "gemini"),
|
||
("claude-opus-4-6-thinking", "claude"),
|
||
("gpt-4o-mini", "gpt"),
|
||
("deepseek-v4-pro", "deepseek"),
|
||
("kimi-k2.7-code", "kimi"),
|
||
("qwen3.8-max", "qwen"),
|
||
("grok-4.5", "grok"),
|
||
("glm-5.3", "glm"),
|
||
]
|
||
for model_name, expected in cases:
|
||
try:
|
||
res = fn(model_name)
|
||
if res != expected:
|
||
return False, f"extract_model_family('{model_name}') = '{res}', expected '{expected}'"
|
||
except Exception as exc:
|
||
return False, f"Exception on '{model_name}': {exc}"
|
||
return True, "All 8 model families correctly extracted"
|
||
|
||
|
||
# Task 2
|
||
def test_t2(fn) -> Tuple[bool, str]:
|
||
try:
|
||
if not fn("secret123", "secret123"):
|
||
return False, "Failed on exact match"
|
||
if fn("secret123", "wrong"):
|
||
return False, "Failed on mismatch (returned True)"
|
||
if fn(None, "secret123"):
|
||
return False, "Failed on None given token"
|
||
if not fn("секретный_токен", "секретный_токен"):
|
||
return False, "Failed on Cyrillic token match"
|
||
if fn("секретный_токен", "другой_токен"):
|
||
return False, "Failed on Cyrillic token mismatch"
|
||
except Exception as exc:
|
||
return False, f"Exception during token verify: {exc}"
|
||
return True, "Constant-time byte comparison passed all cases"
|
||
|
||
|
||
# Task 3
|
||
def test_t3(fn) -> Tuple[bool, str]:
|
||
data = {
|
||
"user": "alice",
|
||
"api_key": "sk-1234567890abcdef",
|
||
"nested": {
|
||
"token": "ghp_secret987654321",
|
||
"safe_url": "https://example.com",
|
||
},
|
||
"log_msg": "Failed: Bearer sk-secret-token-xyz on host",
|
||
}
|
||
try:
|
||
cleaned = fn(data)
|
||
if cleaned["user"] != "alice":
|
||
return False, "Modified safe field 'user'"
|
||
if cleaned["api_key"] != "***":
|
||
return False, f"Failed to mask 'api_key': {cleaned['api_key']}"
|
||
if cleaned["nested"]["token"] != "***":
|
||
return False, f"Failed to mask nested 'token': {cleaned['nested']['token']}"
|
||
if cleaned["nested"]["safe_url"] != "https://example.com":
|
||
return False, "Modified safe nested URL"
|
||
if "sk-secret-token" in str(cleaned["log_msg"]):
|
||
return False, "Leaked inline secret token in log_msg"
|
||
except Exception as exc:
|
||
return False, f"Exception during secret scrub: {exc}"
|
||
return True, "Secret scrubbing passed recursive and string checks"
|
||
|
||
|
||
# Task 4
|
||
def test_t4(cls) -> Tuple[bool, str]:
|
||
try:
|
||
tracker = cls()
|
||
for i in range(3):
|
||
if not tracker.record_edge_traversal("edge-1", max_iterations=3):
|
||
return False, f"Iteration {i+1} of edge-1 was rejected early"
|
||
if tracker.record_edge_traversal("edge-1", max_iterations=3):
|
||
return False, "Iteration 4 of edge-1 was allowed when max was 3"
|
||
if not tracker.record_edge_traversal("edge-2", max_iterations=2):
|
||
return False, "Edge-2 was blocked by edge-1 count"
|
||
except Exception as exc:
|
||
return False, f"Exception in CycleTracker: {exc}"
|
||
return True, "CycleTracker correctly enforced edge iteration limits"
|
||
|
||
|
||
# Task 5
|
||
def test_t5(fn) -> Tuple[bool, str]:
|
||
try:
|
||
allowed_root = "/srv/projects/my-project"
|
||
forbidden = ["agy_profiles", "auth.json", ".ssh"]
|
||
|
||
ok, _ = fn("/srv/projects/my-project/src/main.py", allowed_root, forbidden)
|
||
if not ok:
|
||
return False, "Rejected valid path inside allowed root"
|
||
|
||
ok, _ = fn("/etc/passwd", allowed_root, forbidden)
|
||
if ok:
|
||
return False, "Allowed path outside allowed root (/etc/passwd)"
|
||
|
||
ok, _ = fn("/srv/projects/my-project/../../etc/shadow", allowed_root, forbidden)
|
||
if ok:
|
||
return False, "Allowed path traversal ../../etc/shadow"
|
||
|
||
ok, _ = fn("/srv/projects/my-project/agy_profiles/key.json", allowed_root, forbidden)
|
||
if ok:
|
||
return False, "Allowed forbidden pattern 'agy_profiles'"
|
||
except Exception as exc:
|
||
return False, f"Exception in validate_file_path: {exc}"
|
||
return True, "Path boundary validation passed all containment and forbidden checks"
|
||
|
||
|
||
# Task 6
|
||
def test_t6(fn) -> Tuple[bool, str]:
|
||
try:
|
||
is_dest, cmd, targets = fn("rm -rf /tmp/test_dir /tmp/other")
|
||
if not is_dest or cmd != "rm" or "/tmp/test_dir" not in targets:
|
||
return False, f"Failed on 'rm -rf': got is_dest={is_dest}, cmd={cmd}, targets={targets}"
|
||
|
||
is_dest, cmd, targets = fn("del /f /q C:/temp/file.txt")
|
||
if not is_dest or cmd != "del" or any("file.txt" not in t for t in targets):
|
||
return False, f"Failed on 'del': got is_dest={is_dest}, cmd={cmd}, targets={targets}"
|
||
|
||
is_dest, _, _ = fn("git status")
|
||
if is_dest:
|
||
return False, "Marked safe command 'git status' as destructive"
|
||
|
||
is_dest, _, _ = fn("ls -la /var/log")
|
||
if is_dest:
|
||
return False, "Marked safe command 'ls' as destructive"
|
||
except Exception as exc:
|
||
return False, f"Exception in is_destructive_command: {exc}"
|
||
return True, "Shell command classification passed"
|
||
|
||
|
||
# Task 7
|
||
def test_t7(fn) -> Tuple[bool, str]:
|
||
allowed = {"api.anthropic.com", "api.github.com", "generativelanguage.googleapis.com"}
|
||
try:
|
||
if not fn("https://api.anthropic.com/v1/messages", allowed):
|
||
return False, "Blocked allowed host api.anthropic.com"
|
||
if not fn("http://127.0.0.1:8080/health", allowed):
|
||
return False, "Blocked loopback 127.0.0.1"
|
||
if not fn("http://localhost:11434/api/tags", allowed):
|
||
return False, "Blocked loopback localhost"
|
||
if fn("http://internal-artifactory.local:8081", allowed):
|
||
return False, "Allowed rogue internal host"
|
||
if fn("https://evil-hacker.com/exfil", allowed):
|
||
return False, "Allowed rogue external host"
|
||
except Exception as exc:
|
||
return False, f"Exception in is_outbound_allowed: {exc}"
|
||
return True, "Network whitelist passed"
|
||
|
||
|
||
# Task 8
|
||
def test_t8(fn) -> Tuple[bool, str]:
|
||
try:
|
||
err_comp = {"router_error": True, "error_details": "Failover exhausted", "content": "Raw router error string"}
|
||
res = fn(err_comp, "Fallback: next_call")
|
||
if res.get("content") == "Raw router error string":
|
||
return False, "Router error string was returned as assistant content"
|
||
if res.get("content") != "Fallback: next_call":
|
||
return False, f"Unexpected content: {res.get('content')}"
|
||
if not res.get("router_fallback"):
|
||
return False, "Missing metadata 'router_fallback'"
|
||
|
||
normal_comp = {"router_error": False, "content": "Assistant answer"}
|
||
res_norm = fn(normal_comp, "Fallback")
|
||
if res_norm.get("content") != "Assistant answer":
|
||
return False, "Normal response was corrupted"
|
||
except Exception as exc:
|
||
return False, f"Exception in sanitize_hermes_response: {exc}"
|
||
return True, "Router safety fuse response sanitization passed"
|
||
|
||
|
||
# Task 9
|
||
def test_t9(fn) -> Tuple[bool, str]:
|
||
try:
|
||
r, src = fn(explicit_role="developer-1", model="claude-opus", session_role="manager")
|
||
if r != "developer-1" or src != "explicit":
|
||
return False, f"Level 1 failed: got ({r}, {src}), expected ('developer-1', 'explicit')"
|
||
|
||
r, src = fn(explicit_role=None, model="claude-opus-4-6", session_role="manager")
|
||
if r != "code-reviewer" or src != "model_match":
|
||
return False, f"Level 2 failed: got ({r}, {src}), expected ('code-reviewer', 'model_match')"
|
||
|
||
r, src = fn(explicit_role=None, model="unknown-model", session_role="developer-2")
|
||
if r != "developer-2" or src != "session_affinity":
|
||
return False, f"Level 3 failed: got ({r}, {src}), expected ('developer-2', 'session_affinity')"
|
||
|
||
r, src = fn(explicit_role=None, model="unknown-model", session_role=None, default_role="manager")
|
||
if r != "manager" or src != "default_fallback":
|
||
return False, f"Level 4 failed: got ({r}, {src}), expected ('manager', 'default_fallback')"
|
||
except Exception as exc:
|
||
return False, f"Exception in resolve_role: {exc}"
|
||
return True, "4-level role resolution passed in strict hierarchy"
|
||
|
||
|
||
# Task 10
|
||
def test_t10(fn) -> Tuple[bool, str]:
|
||
base = {
|
||
"PATH": "/usr/bin:/bin",
|
||
"HOME": "/home/user",
|
||
"OPENAI_API_KEY": "sk-secret123",
|
||
"MY_TOKEN": "token_val",
|
||
"LANG": "en_US.UTF-8",
|
||
}
|
||
allowed = {"PATH", "HOME", "LANG", "OPENAI_API_KEY"}
|
||
overrides = {"USERPROFILE": "/srv/profile_1"}
|
||
|
||
try:
|
||
clean = fn(base, allowed, overrides)
|
||
if "PATH" not in clean or clean["PATH"] != "/usr/bin:/bin":
|
||
return False, "Missing allowed 'PATH'"
|
||
if "OPENAI_API_KEY" in clean:
|
||
return False, "Leaked 'OPENAI_API_KEY' despite being in allowed list"
|
||
if "MY_TOKEN" in clean:
|
||
return False, "Leaked 'MY_TOKEN'"
|
||
if clean.get("USERPROFILE") != "/srv/profile_1":
|
||
return False, "Override 'USERPROFILE' was not applied"
|
||
except Exception as exc:
|
||
return False, f"Exception in build_safe_env: {exc}"
|
||
return True, "Safe environment constructor passed"
|
||
|
||
|
||
# Task 11
|
||
def test_t11(fn) -> Tuple[bool, str]:
|
||
try:
|
||
if fn(is_enabled=False, is_authenticated=True, is_auth_expired=False, cooldown_sec=0, is_cold_spare=False) != "disabled":
|
||
return False, "Failed disabled check"
|
||
if fn(is_enabled=True, is_authenticated=False, is_auth_expired=True, cooldown_sec=0, is_cold_spare=False) != "auth_expired":
|
||
return False, "Failed auth_expired check"
|
||
if fn(is_enabled=True, is_authenticated=False, is_auth_expired=False, cooldown_sec=0, is_cold_spare=True) != "cold_spare":
|
||
return False, "Failed cold_spare check"
|
||
if fn(is_enabled=True, is_authenticated=False, is_auth_expired=False, cooldown_sec=0, is_cold_spare=False) != "not_configured":
|
||
return False, "Failed not_configured check"
|
||
if fn(is_enabled=True, is_authenticated=True, is_auth_expired=False, cooldown_sec=120, is_cold_spare=False) != "quota_exhausted":
|
||
return False, "Failed quota_exhausted check"
|
||
if fn(is_enabled=True, is_authenticated=True, is_auth_expired=False, cooldown_sec=0, is_cold_spare=False) != "healthy":
|
||
return False, "Failed healthy check"
|
||
except Exception as exc:
|
||
return False, f"Exception in determine_profile_health: {exc}"
|
||
return True, "Unified health status priority passed"
|
||
|
||
|
||
# Task 12
|
||
def test_t12(cls) -> Tuple[bool, str]:
|
||
try:
|
||
lm = cls(default_max_concurrency=2, default_lease_timeout=5.0)
|
||
l1 = lm.acquire("profile-1")
|
||
if not l1.get("granted"):
|
||
return False, "Failed to acquire first lease for profile-1"
|
||
l2 = lm.acquire("profile-1")
|
||
if not l2.get("granted"):
|
||
return False, "Failed to acquire second lease for profile-1"
|
||
l3 = lm.acquire("profile-1")
|
||
if l3.get("granted"):
|
||
return False, "Granted 3rd lease when max concurrency was 2"
|
||
|
||
lm.release("profile-1", l1["lease_id"])
|
||
l4 = lm.acquire("profile-1")
|
||
if not l4.get("granted"):
|
||
return False, "Failed to acquire lease after release"
|
||
except Exception as exc:
|
||
return False, f"Exception in Long-Context LeaseManager: {exc}"
|
||
return True, "Long-context LeaseManager correctly implemented concurrent slots and release"
|
||
|
||
|
||
BENCHMARK_TASKS: List[BenchmarkTask] = [
|
||
BenchmarkTask(
|
||
task_id="T01_extract_model_family",
|
||
title="Extract Model Family",
|
||
category="routing",
|
||
prompt="Write a Python function `extract_model_family(model_name: str) -> str` that inspects a model identifier string (e.g. 'gemini-3.7-flash', 'claude-opus-4-6', 'gpt-4o', 'deepseek-v4-pro', 'kimi-k2.7', 'qwen3.8-max', 'grok-4.5', 'glm-5.3') and returns the canonical lower-case family name ('gemini', 'claude', 'gpt', 'deepseek', 'kimi', 'qwen', 'grok', 'glm'). If no family is recognized, return 'unknown'. Output only the Python code without extra conversational text.",
|
||
expected_function_name="extract_model_family",
|
||
test_function=test_t1,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T02_verify_auth_token",
|
||
title="Constant-Time Byte Token Comparison",
|
||
category="security",
|
||
prompt="Write a Python function `verify_auth_token(given_token: str | None, required_token: str) -> bool` that performs a constant-time byte-level comparison using `secrets.compare_digest`. It must handle `given_token` being `None` or non-ASCII characters without raising `TypeError`. Return `True` if tokens match, `False` otherwise. Output only the Python code without extra conversational text.",
|
||
expected_function_name="verify_auth_token",
|
||
test_function=test_t2,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T03_scrub_secrets",
|
||
title="Recursive Secret & PII Scrubbing",
|
||
category="security",
|
||
prompt="Write a Python function `scrub_secrets(data: Any) -> Any` that recursively processes dictionaries, lists, and strings. If a dictionary key contains (case-insensitive) 'token', 'secret', 'api_key', 'password', or 'bearer', its value must be replaced with '***'. Strings containing patterns like 'Bearer <token>' or 'sk-<token>' must have the token replaced with '***'. All other keys and values must be preserved intact. Output only the Python code.",
|
||
expected_function_name="scrub_secrets",
|
||
test_function=test_t3,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T04_cycle_tracker",
|
||
title="DAG Loop & Cycle Iteration Tracker",
|
||
category="workflow",
|
||
prompt="Write a Python class `CycleTracker` with method `record_edge_traversal(self, edge_id: str, max_iterations: int) -> bool`. It tracks traversal counts per `edge_id`. If traversal count <= max_iterations, return `True`. If it exceeds max_iterations, return `False`. Output only the Python code.",
|
||
expected_function_name="CycleTracker",
|
||
test_function=test_t4,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T05_validate_file_path",
|
||
title="Safe File Path Boundary Validation",
|
||
category="security",
|
||
prompt="Write a Python function `validate_file_path(target_path: str, allowed_root: str, forbidden_patterns: list[str]) -> tuple[bool, str]` using `pathlib.Path`. Resolve both paths to prevent `../` traversal attacks. Ensure `target_path` is strictly within `allowed_root`. If any forbidden pattern (case-insensitive substring) is present in the path, return `(False, 'Forbidden path pattern')`. If outside allowed root, return `(False, 'Path outside boundary')`. Otherwise return `(True, 'OK')`. Output only the Python code.",
|
||
expected_function_name="validate_file_path",
|
||
test_function=test_t5,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T06_is_destructive_command",
|
||
title="Shell Command Classifier",
|
||
category="security",
|
||
prompt="Write a Python function `is_destructive_command(cmd_line: str) -> tuple[bool, str, list[str]]`. Parse the command line (using `shlex.split`). If the executable is in {'rm', 'rmdir', 'unlink', 'del', 'erase', 'remove-item', 'rd'}, return `(True, command_name, list_of_target_paths)` filtering out argument flags (e.g. starting with '-' or Windows flags like '/f', '/q'). Otherwise return `(False, command_name, [])`. Output only the Python code.",
|
||
expected_function_name="is_destructive_command",
|
||
test_function=test_t6,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T07_is_outbound_allowed",
|
||
title="Outbound Destination Network Whitelist",
|
||
category="network",
|
||
prompt="Write a Python function `is_outbound_allowed(url_or_host: str, allowed_hosts: set[str]) -> bool` using `urllib.parse`. Extract the hostname in lower-case. Return `True` if hostname is in `allowed_hosts`, is a subdomain of an allowed host, or is loopback ('127.0.0.1', 'localhost'). Otherwise return `False`. Output only the Python code.",
|
||
expected_function_name="is_outbound_allowed",
|
||
test_function=test_t7,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T08_sanitize_hermes_response",
|
||
title="Router Safety Fuse Response Sanitizer",
|
||
category="router",
|
||
prompt="Write a Python function `sanitize_hermes_response(completion: dict, fallback_message: str) -> dict`. If `completion.get('router_error')` is True, replace `content` with `fallback_message` and set `router_fallback: True`. Otherwise return a copy of `completion` with original `content`. Never allow raw router errors to become assistant content. Output only the Python code.",
|
||
expected_function_name="sanitize_hermes_response",
|
||
test_function=test_t8,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T09_resolve_role",
|
||
title="4-Level Dynamic Role Resolver",
|
||
category="router",
|
||
prompt="Write a Python function `resolve_role(explicit_role: str | None, model: str | None, session_role: str | None, default_role: str = 'manager') -> tuple[str, str]`. Resolve role strictly in 4 hierarchical levels: 1. `explicit_role` -> return (explicit_role, 'explicit'); 2. `model` contains 'claude' -> return ('code-reviewer', 'model_match'), 'gemini-3.1' -> ('developer-2', 'model_match'), 'gemini-3.7' -> ('developer-1', 'model_match'); 3. `session_role` -> return (session_role, 'session_affinity'); 4. return (default_role, 'default_fallback'). Never use regex prompt guessing. Output only the Python code.",
|
||
expected_function_name="resolve_role",
|
||
test_function=test_t9,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T10_build_safe_env",
|
||
title="Isolated Subprocess Environment Constructor",
|
||
category="security",
|
||
prompt="Write a Python function `build_safe_env(base_env: dict[str, str], allowed_keys: set[str], overrides: dict[str, str]) -> dict[str, str]`. Copy only keys present in `allowed_keys`. Strip any key containing (case-insensitive) 'api_key', 'token', 'secret', or 'password'. Apply `overrides` at the end. Output only the Python code.",
|
||
expected_function_name="build_safe_env",
|
||
test_function=test_t10,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T11_determine_profile_health",
|
||
title="Unified Health Status Priority Resolver",
|
||
category="health",
|
||
prompt="Write a Python function `determine_profile_health(is_enabled: bool, is_authenticated: bool, is_auth_expired: bool, cooldown_sec: int, is_cold_spare: bool) -> str`. Resolve status in exact priority: 1. not is_enabled -> 'disabled'; 2. not is_authenticated: if is_auth_expired -> 'auth_expired', elif is_cold_spare -> 'cold_spare', else -> 'not_configured'; 3. cooldown_sec > 0 -> 'quota_exhausted'; 4. else -> 'healthy'. Output only the Python code.",
|
||
expected_function_name="determine_profile_health",
|
||
test_function=test_t11,
|
||
),
|
||
BenchmarkTask(
|
||
task_id="T12_long_context_lease_manager",
|
||
title="Long Context (32k+) Thread-Safe Lease Manager",
|
||
category="concurrency",
|
||
prompt="Write a Python class `LeaseManager` with `__init__(self, default_max_concurrency: int = 2, default_lease_timeout: float = 30.0)`, `acquire(self, profile_id: str, max_concurrency: int | None = None) -> dict`, and `release(self, profile_id: str, lease_id: str) -> bool`. `acquire` returns `{'granted': True, 'lease_id': lid, 'active_count': int}` if current active leases < max_concurrency, else `{'granted': False, 'active_count': int}`. `release` removes the lease by `lease_id` and returns `True` if found. Ensure thread-safety using `threading.Lock`. Output only the Python code.",
|
||
expected_function_name="LeaseManager",
|
||
test_function=test_t12,
|
||
is_long_context=True,
|
||
),
|
||
]
|