b4ae08ef53
Preflight and updater tests no longer probe 8081/8082 or GitHub. Hermetic runs fail-fast on non-loopback sockets. GUI helpers are importable without customtkinter. CI installs the web extra, and pytest-timeout plus a wall-clock wrapper bound the suite.
272 lines
9.1 KiB
Python
272 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Per-file and per-node pytest hang diagnostic runner.
|
|
|
|
Each file (or node) is a separate subprocess with a hard timeout.
|
|
One hang never blocks the rest of the suite.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_OUT = ROOT / "artifacts" / "test-diagnostics"
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def list_test_files() -> list[Path]:
|
|
tests_dir = ROOT / "tests"
|
|
return sorted(p for p in tests_dir.glob("test_*.py") if p.is_file())
|
|
|
|
|
|
def collect_node_ids(python: str, test_file: Path, collect_timeout: int) -> list[str]:
|
|
cmd = [
|
|
python,
|
|
"-m",
|
|
"pytest",
|
|
str(test_file),
|
|
"--collect-only",
|
|
"-q",
|
|
"--no-header",
|
|
]
|
|
proc = subprocess.run(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=collect_timeout,
|
|
env=os.environ.copy(),
|
|
)
|
|
nodes: list[str] = []
|
|
for line in (proc.stdout or "").splitlines():
|
|
line = line.strip()
|
|
if line.startswith(str(test_file).replace("\\", "/")) or line.startswith("tests/"):
|
|
if "::" in line and not line.startswith("="):
|
|
nodes.append(line.split()[0])
|
|
elif "::" in line and not line.startswith("=") and "error" not in line.lower():
|
|
if line.startswith("test_") or "/test_" in line or line.startswith("tests"):
|
|
nodes.append(line.split()[0])
|
|
# Fallback: pytest -q collect prints node ids as first token.
|
|
if not nodes:
|
|
for line in (proc.stdout or "").splitlines():
|
|
stripped = line.strip()
|
|
if "::" in stripped and not stripped.startswith("="):
|
|
nodes.append(stripped.split()[0])
|
|
return nodes
|
|
|
|
|
|
def _kill_tree(proc: subprocess.Popen) -> None:
|
|
if proc.poll() is not None:
|
|
return
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
try:
|
|
proc.kill()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
|
|
|
|
def run_guarded(
|
|
python: str,
|
|
target: str,
|
|
timeout_s: int,
|
|
log_dir: Path,
|
|
extra_args: list[str] | None = None,
|
|
) -> dict:
|
|
safe_name = target.replace("/", "_").replace("::", "__").replace("[", "_").replace("]", "_")
|
|
stdout_path = log_dir / f"{safe_name}.stdout.txt"
|
|
stderr_path = log_dir / f"{safe_name}.stderr.txt"
|
|
dump_path = log_dir / f"{safe_name}.faulthandler.txt"
|
|
cmd = [
|
|
python,
|
|
"-X",
|
|
"faulthandler",
|
|
"-m",
|
|
"pytest",
|
|
target,
|
|
"-vv",
|
|
"--tb=short",
|
|
"-p",
|
|
"no:cacheprovider",
|
|
]
|
|
if extra_args:
|
|
cmd.extend(extra_args)
|
|
env = os.environ.copy()
|
|
env["PYTHONFAULTHANDLER"] = "1"
|
|
start = time.monotonic()
|
|
start_iso = utc_now()
|
|
status = "UNKNOWN"
|
|
return_code: int | None = None
|
|
timed_out = False
|
|
dump = ""
|
|
stdout_text = ""
|
|
stderr_text = ""
|
|
try:
|
|
with open(stdout_path, "w", encoding="utf-8") as out_f, open(
|
|
stderr_path, "w", encoding="utf-8"
|
|
) as err_f:
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
stdout=out_f,
|
|
stderr=err_f,
|
|
text=True,
|
|
start_new_session=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
return_code = proc.wait(timeout=timeout_s)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGABRT)
|
|
time.sleep(0.4)
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
pass
|
|
_kill_tree(proc)
|
|
return_code = -9
|
|
except Exception:
|
|
dump = traceback.format_exc()
|
|
status = "ERROR"
|
|
return_code = -1
|
|
duration = round(time.monotonic() - start, 3)
|
|
try:
|
|
stdout_text = stdout_path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
stdout_text = ""
|
|
try:
|
|
stderr_text = stderr_path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
stderr_text = ""
|
|
if timed_out:
|
|
status = "TIMEOUT"
|
|
dump_parts = [dump, "=== STDERR TAIL ===\n" + stderr_text[-8000:], "=== STDOUT TAIL ===\n" + stdout_text[-8000:]]
|
|
dump = "\n".join(p for p in dump_parts if p)
|
|
dump_path.write_text(dump, encoding="utf-8")
|
|
elif return_code == 0:
|
|
status = "PASS"
|
|
else:
|
|
status = "FAIL"
|
|
dump_path.write_text(
|
|
(stderr_text[-8000:] + "\n" + stdout_text[-8000:]),
|
|
encoding="utf-8",
|
|
)
|
|
return {
|
|
"target": target,
|
|
"start_time": start_iso,
|
|
"duration": duration,
|
|
"return_code": return_code,
|
|
"status": status,
|
|
"stdout_path": str(stdout_path),
|
|
"stderr_path": str(stderr_path),
|
|
"dump_path": str(dump_path) if dump_path.exists() else None,
|
|
"timed_out": timed_out,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--python", default=sys.executable)
|
|
parser.add_argument("--file-timeout", type=int, default=90)
|
|
parser.add_argument("--node-timeout", type=int, default=30)
|
|
parser.add_argument("--collect-timeout", type=int, default=30)
|
|
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
|
parser.add_argument("--mode", choices=["files", "nodes"], default="files")
|
|
parser.add_argument("--file", action="append", default=[])
|
|
args = parser.parse_args()
|
|
out_dir = args.out
|
|
logs = out_dir / ("file-logs" if args.mode == "files" else "node-logs")
|
|
logs.mkdir(parents=True, exist_ok=True)
|
|
if args.file:
|
|
files = [Path(f) if Path(f).is_absolute() else ROOT / f for f in args.file]
|
|
else:
|
|
files = list_test_files()
|
|
results: list[dict] = []
|
|
if args.mode == "files":
|
|
for path in files:
|
|
rel = str(path.relative_to(ROOT)) if path.is_absolute() else str(path)
|
|
print(f"[RUN FILE] {rel}", flush=True)
|
|
rec = run_guarded(args.python, rel, args.file_timeout, logs)
|
|
rec["file"] = rel
|
|
results.append(rec)
|
|
print(f" -> {rec['status']} {rec['duration']}s rc={rec['return_code']}", flush=True)
|
|
payload = {
|
|
"generated_at": utc_now(),
|
|
"mode": "files",
|
|
"file_timeout": args.file_timeout,
|
|
"results": results,
|
|
"summary": {
|
|
"total": len(results),
|
|
"pass": sum(1 for r in results if r["status"] == "PASS"),
|
|
"fail": sum(1 for r in results if r["status"] == "FAIL"),
|
|
"timeout": sum(1 for r in results if r["status"] == "TIMEOUT"),
|
|
},
|
|
}
|
|
out_path = out_dir / "file-results.json"
|
|
else:
|
|
for path in files:
|
|
rel = str(path.relative_to(ROOT)) if path.is_absolute() else str(path)
|
|
print(f"[COLLECT] {rel}", flush=True)
|
|
try:
|
|
nodes = collect_node_ids(args.python, Path(rel), args.collect_timeout)
|
|
except subprocess.TimeoutExpired:
|
|
results.append(
|
|
{
|
|
"file": rel,
|
|
"target": rel,
|
|
"status": "COLLECT_TIMEOUT",
|
|
"start_time": utc_now(),
|
|
"duration": args.collect_timeout,
|
|
"return_code": -9,
|
|
"timed_out": True,
|
|
}
|
|
)
|
|
continue
|
|
if not nodes:
|
|
print(f" no nodes collected for {rel}", flush=True)
|
|
continue
|
|
for node in nodes:
|
|
print(f"[RUN NODE] {node}", flush=True)
|
|
rec = run_guarded(args.python, node, args.node_timeout, logs)
|
|
rec["file"] = rel
|
|
rec["node_id"] = node
|
|
results.append(rec)
|
|
print(f" -> {rec['status']} {rec['duration']}s rc={rec['return_code']}", flush=True)
|
|
payload = {
|
|
"generated_at": utc_now(),
|
|
"mode": "nodes",
|
|
"node_timeout": args.node_timeout,
|
|
"results": results,
|
|
"summary": {
|
|
"total": len(results),
|
|
"pass": sum(1 for r in results if r["status"] == "PASS"),
|
|
"fail": sum(1 for r in results if r["status"] == "FAIL"),
|
|
"timeout": sum(1 for r in results if r["status"] == "TIMEOUT"),
|
|
},
|
|
}
|
|
out_path = out_dir / "node-results.json"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
print(json.dumps(payload["summary"], indent=2), flush=True)
|
|
print(f"wrote {out_path}", flush=True)
|
|
return 0 if payload["summary"]["timeout"] == 0 else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|