diff --git a/.scratch/v111-architecture-scan.md b/.scratch/v111-architecture-scan.md index 22199d1..520375a 100644 --- a/.scratch/v111-architecture-scan.md +++ b/.scratch/v111-architecture-scan.md @@ -113,6 +113,34 @@ except Exception as e: **当前状态**:两个 xfail 守卫在 `test_openclaw_client_and_webhook.py`,CI 不阻塞;修复后移除 xfail 即转绿。 +### 6.3 前端 CronPoolMonitor 不响应 visibilitychange(S6 缺口,T2 暴露) + +**症状**:`frontend/web/src/pages/CronPoolMonitor.test.tsx` 的 `it.fails('listens to visibilitychange ...')`。 +CronPoolMonitor 当前的 `useEffect` 只做了 `setInterval(loadData, 5000)` + `clearInterval` 清理,不监听 `document.visibilitychange`。 + +**影响**:当浏览器标签页被切换到后台或最小化时,仍每 5 秒轮询后端 `/api/openclaw/cron-pool/*` —— 浪费网络/CPU,且与 `useIntelligentEvalRead` 的"终态停轮询"行为漂移。 + +**修复方向**(待开独立 issue,不在 #6 范围): +- 在 `useEffect` 里 `document.addEventListener('visibilitychange', ...)`;`hidden` 时清除 timer 并清掉 `interval` 引用,`visible` 时立即 `loadData()` 并重建 interval;cleanup 同时 `removeEventListener`。 +- 或抽统一 `usePolling(loader, { intervalMs, pauseWhenHidden })` hook 让智能评估读模型与 Cron 池共用。 + +**当前状态**:`it.fails` 守卫在 `CronPoolMonitor.test.tsx`,CI 显示 `1 expected fail`(vitest 等价 xfail),不阻塞;修复后移除 `it.fails` 即转绿。 + +### 6.4 无告警→auto_scale 联动(真缺口,T7 暴露) + +**症状**:`tests/integration/test_alert_autoscale_link.py::test_check_alerts_triggers_auto_scale_on_high_utilization` xfail。 +`backend/agenteval/web/routers/openclaw_cron_pool.py::check_alerts` 走完整条规则链只创建 `AlertHistoryDB` 行,**从不调** `cron_pool.scale_up` / `scale_down`。高利用率告警入库后无人(或外部 cron)触发 `auto_scale`,生产需要操作员看 Web UI 手动点。 + +**注意(已修正的早期误判)**:`AlertManager` 在 router 内 per-request 新建看似 bug,但 `check_rules` 逻辑对 `duration_minutes==0` 的规则(`task_backlog`、`stuck_rate`)走 fall-through 分支直接创建 alert,不需要 `triggered_at` 持久化。`task_backlog` 端到端测试 xpass 即证明。`pool_utilization`(duration=10)若端到端测试,会暴露 per-request `triggered_at` 丢失,但当前 alert 联动缺口更优先。 + +**影响**:高利用率/任务积压/卡死率告警频发但扩缩容不会自动发生(依赖人盯),或发生时机滞后。 + +**修复方向**(待开独立 issue,不在 #6 范围): +- `check_alerts` 在创建 alert 后按 `alert_type` 映射调用 `cron_pool.scale_up` / `scale_down`;或注册 webhook 接收方;或独立 worker 轮询 `unresolved alerts` → 调 `auto_scale`。 +- 顺带把 `pool_utilization`(duration_minutes=10)规则的 `triggered_at` 持久化(如果后续要端到端测持续窗口)。 + +**当前状态**:1 xfail 守卫在 `test_alert_autoscale_link.py`,CI 不阻塞;修复后移除 xfail 即转绿。 + ### 薄弱/缺失点(加固清单 T) ### 薄弱/缺失点(加固清单 T) diff --git a/frontend/web/src/pages/CronPoolMonitor.test.tsx b/frontend/web/src/pages/CronPoolMonitor.test.tsx new file mode 100644 index 0000000..6270fb4 --- /dev/null +++ b/frontend/web/src/pages/CronPoolMonitor.test.tsx @@ -0,0 +1,110 @@ +// @vitest-environment jsdom +import { act, cleanup, render } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('../api', () => ({ + openclawCronPoolApi: { + getStatus: vi.fn(), + getMetrics: vi.fn(), + getAlerts: vi.fn(), + scale: vi.fn(), + resolveAlert: vi.fn(), + }, +})) + +vi.mock('antd', async () => { + const actual = await vi.importActual('antd') + return { + ...actual, + message: { error: vi.fn(), success: vi.fn(), warning: vi.fn(), info: vi.fn() }, + } +}) + +import { openclawCronPoolApi } from '../api' +import CronPoolMonitor from './CronPoolMonitor' + +const statusResp = { + data: { pool: { total: 5, idle: 3, busy: 2, stuck: 0, min_size: 5, max_size: 20 } }, +} +const metricsResp = { data: { metrics: { pool_utilization: 0.4, task_backlog: 0, stuck_rate: 0 } } } +const alertsResp = { data: { alerts: [] } } + +beforeEach(() => { + Object.defineProperty(window, 'matchMedia', { + writable: true, + value: vi.fn().mockImplementation((query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), + removeListener: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), + }) + vi.useFakeTimers() + vi.mocked(openclawCronPoolApi.getStatus).mockResolvedValue(statusResp as never) + vi.mocked(openclawCronPoolApi.getMetrics).mockResolvedValue(metricsResp as never) + vi.mocked(openclawCronPoolApi.getAlerts).mockResolvedValue(alertsResp as never) +}) + +afterEach(() => { + vi.useRealTimers() + cleanup() +}) + +async function settle() { + await act(async () => { + await Promise.resolve() + }) +} + +describe('CronPoolMonitor polling lifecycle', () => { + it('polls every 5 seconds while mounted', async () => { + render() + await settle() + expect(openclawCronPoolApi.getStatus).toHaveBeenCalledTimes(1) + await act(async () => { + await vi.advanceTimersByTimeAsync(5000) + }) + await settle() + expect(openclawCronPoolApi.getStatus).toHaveBeenCalledTimes(2) + await act(async () => { + await vi.advanceTimersByTimeAsync(5000) + }) + await settle() + expect(openclawCronPoolApi.getStatus).toHaveBeenCalledTimes(3) + }) + + it('clears the polling interval on unmount', async () => { + const { unmount } = render() + await settle() + await act(async () => { + await vi.advanceTimersByTimeAsync(5000) + }) + await settle() + const before = vi.mocked(openclawCronPoolApi.getStatus).mock.calls.length + unmount() + await act(async () => { + await Promise.resolve() + await Promise.resolve() + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(30000) + }) + await settle() + const after = vi.mocked(openclawCronPoolApi.getStatus).mock.calls.length + expect(after).toBe(before) + }) + it.fails('listens to visibilitychange to pause polling when tab is hidden', async () => { + const addSpy = vi.spyOn(document, 'addEventListener') + render() + await settle() + const hasVisibility = addSpy.mock.calls.some( + ([event]: [string]) => event === 'visibilitychange', + ) + expect(hasVisibility).toBe(true) + addSpy.mockRestore() + }) +}) diff --git a/tests/integration/test_alert_autoscale_link.py b/tests/integration/test_alert_autoscale_link.py new file mode 100644 index 0000000..c1f95e7 --- /dev/null +++ b/tests/integration/test_alert_autoscale_link.py @@ -0,0 +1,166 @@ +"""Alert → auto-scale end-to-end link (Gitea issue #6 / T7). + +The desired end-to-end behaviour: a sustained high-utilization alert must +provoke an auto-scale-up. Today the two paths are independent — `check_alerts` +records alerts but never invokes `auto_scale`. This test guards the desired +end-to-end loop; the xfail marks the missing link. +""" +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine, select + +from agenteval.storage.db import IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + IntelligentEvalDB, + IntelligentEvalSessionDB, + IntelligentEvalTaskQueueDB, + ) + + engine = create_engine( + f"sqlite:///{tmp_path / 'test.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + yield TestClient(app) + app.dependency_overrides.clear() + session.close() + engine.dispose() + + +@pytest.fixture() +def db_session(client): + return next(app.dependency_overrides[get_db]()) + + +def _seed_high_utilization_pool(db_session: Session) -> None: + """10 crons: 9 busy + 1 idle => pool_utilization = 0.9 (rule threshold).""" + for i in range(10): + db_session.add( + OpenClawCronPoolDB( + openclaw_cron_id=f"cron-{i}", + status="busy" if i < 9 else "idle", + last_active_at=utc_now(), + ) + ) + db_session.commit() + + +def test_check_alerts_records_task_backlog_alert( + client: TestClient, db_session: Session +): + """task_backlog threshold = 50 with duration_minutes = 0 must fire on call. + + Currently fails: AlertManager is per-request — triggered_at resets every + request, so even duration_minutes=0 rules never re-fire on subsequent + calls. Tracked in .scratch/v111-architecture-scan.md. + """ + from agenteval.storage.db import IntelligentEvalDB + eval_db = IntelligentEvalDB( + name="backlog-eval", + target_id="t1", + status="executing", + started_at=utc_now(), + ) + db_session.add(eval_db) + db_session.commit() + for i in range(51): + db_session.add( + IntelligentEvalTaskQueueDB( + eval_id=eval_db.id, + status="pending", + priority=1, + reason="slot_due", + ) + ) + db_session.commit() + + r = client.post("/api/openclaw/cron-pool/check-alerts") + assert r.status_code == 200 + types = {a["alert_type"] for a in r.json().get("alerts", [])} + assert "task_backlog" in types + + +@pytest.mark.xfail( + reason=( + "Known gap: no alert→auto-scale link. check_alerts records alerts but " + "never invokes auto_scale. An operator must observe the alert and call " + "/cron-pool/auto-scale manually. Tracked in " + ".scratch/v111-architecture-scan.md." + ), + strict=False, +) +def test_check_alerts_triggers_auto_scale_on_high_utilization( + client: TestClient, db_session: Session +): + """check_alerts must drive auto_scale so high-utilization triggers a scale-up. + + Today the link is missing: check_alerts is independent of auto_scale. + """ + _seed_high_utilization_pool(db_session) + + from agenteval.intelligent_eval import cron_pool as cron_pool_mod + + real_scale_up = cron_pool_mod.scale_up + cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment] + try: + client.post("/api/openclaw/cron-pool/check-alerts") + client.post("/api/openclaw/cron-pool/check-alerts") + + # If the link existed, the alert would have driven a scale_up call. + # Today: no such call. The test asserts the desired behaviour. + assert cron_pool_mod.scale_up.await_count >= 1 + finally: + cron_pool_mod.scale_up = real_scale_up + + +@pytest.mark.xfail( + reason=( + "Known gap: no alert→auto-scale link. check_alerts records alerts but " + "never invokes auto_scale. An operator must observe the alert and call " + "/cron-pool/auto-scale manually. Tracked in " + ".scratch/v111-architecture-scan.md." + ), + strict=False, +) +def test_check_alerts_triggers_auto_scale_on_high_utilization( + client: TestClient, db_session: Session +): + """check_alerts must drive auto_scale so high-utilization triggers a scale-up. + + Today the link is missing: check_alerts is independent of auto_scale. + """ + _seed_high_utilization_pool(db_session) + + # Mock the OpenClaw client so auto_scale can call scale_up without + # actually shelling out. + from agenteval.intelligent_eval import cron_pool as cron_pool_mod + + real_scale_up = cron_pool_mod.scale_up + cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment] + try: + # Trigger the alert path twice (first sets triggered_at, second fires). + client.post("/api/openclaw/cron-pool/check-alerts") + client.post("/api/openclaw/cron-pool/check-alerts") + + # If the link existed, the alert would have driven a scale_up call. + # Today: no such call. The test asserts the desired behaviour. + assert cron_pool_mod.scale_up.await_count >= 1 + finally: + cron_pool_mod.scale_up = real_scale_up diff --git a/tests/integration/test_decision_log_immutability.py b/tests/integration/test_decision_log_immutability.py new file mode 100644 index 0000000..294f545 --- /dev/null +++ b/tests/integration/test_decision_log_immutability.py @@ -0,0 +1,143 @@ +"""Decision-log immutability & dedup contract (Gitea issue #6 / T8). + +Pins the desired behaviour: a decision log is append-only and not silently +overwritten or duplicated. The current router writes through directly; once +it moves into a service (P3), these guards must continue to pass. +""" +import uuid + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine, select + +from agenteval.intelligent_eval.models import IntelligentEvalStatus +from agenteval.storage.db import ( + IntelligentEvalDB, + IntelligentEvalDecisionLogDB, + utc_now, +) +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + IntelligentEvalDB, + IntelligentEvalSessionDB, + IntelligentEvalTaskQueueDB, + ) + + engine = create_engine( + f"sqlite:///{tmp_path / 'test.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + yield TestClient(app) + app.dependency_overrides.clear() + session.close() + engine.dispose() + + +@pytest.fixture() +def db_session(client): + return next(app.dependency_overrides[get_db]()) + + +def _make_eval(db_session: Session) -> IntelligentEvalDB: + eval_db = IntelligentEvalDB( + id=str(uuid.uuid4()), + name="eval-dl-immut", + target_id="t1", + status=IntelligentEvalStatus.EXECUTING.value, + ) + db_session.add(eval_db) + db_session.commit() + return eval_db + + +def _post_log(client: TestClient, eval_id: str, **overrides) -> dict: + body = { + "decision_type": "execute_session", + "reason": "slot_due", + "context": {"slot": "8-10h", "deficit": 2}, + "cron_id": "cron-1", + } + body.update(overrides) + response = client.post( + f"/api/intelligent-evals/{eval_id}/decision-logs", + json=body, + ) + assert response.status_code == 200, response.text + return response.json() + + +def test_decision_log_is_append_only_on_context_change( + client: TestClient, db_session: Session +): + """Modifying context must append, never overwrite, an existing log row.""" + eval_db = _make_eval(db_session) + + first = _post_log( + client, eval_db.id, context={"slot": "8-10h", "deficit": 2} + ) + second = _post_log( + client, eval_db.id, context={"slot": "10-12h", "deficit": 1} + ) + + assert first["id"] != second["id"] + + rows = db_session.exec( + select(IntelligentEvalDecisionLogDB) + .where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id) + .order_by(IntelligentEvalDecisionLogDB.created_at) + ).all() + assert len(rows) == 2 + # The first row's context is preserved (not overwritten by the second). + assert rows[0].get_context() == {"slot": "8-10h", "deficit": 2} + assert rows[1].get_context() == {"slot": "10-12h", "deficit": 1} + + +@pytest.mark.xfail( + reason=( + "Known gap: decision-logs are appended on every call regardless of " + "(eval_id, decision_type, context) identity — i.e. no dedupe. The same " + "decision made twice produces two identical rows. Tracked in " + ".scratch/v111-architecture-scan.md." + ), + strict=False, +) +def test_decision_log_dedupes_identical_entries( + client: TestClient, db_session: Session +): + """Identical (decision_type, context) must not insert a second row.""" + eval_db = _make_eval(db_session) + + body = { + "decision_type": "execute_session", + "reason": "slot_due", + "context": {"slot": "8-10h", "deficit": 2}, + "cron_id": "cron-1", + } + for _ in range(3): + r = client.post( + f"/api/intelligent-evals/{eval_db.id}/decision-logs", + json=body, + ) + assert r.status_code == 200 + + rows = db_session.exec( + select(IntelligentEvalDecisionLogDB).where( + IntelligentEvalDecisionLogDB.eval_id == eval_db.id + ) + ).all() + assert len(rows) == 1, f"expected dedup, got {len(rows)} rows" diff --git a/tests/integration/test_router_orm_guards.py b/tests/integration/test_router_orm_guards.py new file mode 100644 index 0000000..bcca3ec --- /dev/null +++ b/tests/integration/test_router_orm_guards.py @@ -0,0 +1,163 @@ +"""Router ORM contract guards (Gitea issue #6 / T3). + +These tests pin the observable behaviour of router endpoints that currently +embed ORM writes directly (S2). When those handlers are later moved into a +service, the tests should still pass unchanged — that is the contract. +""" +from datetime import timedelta + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine, select + +from agenteval.intelligent_eval.models import IntelligentEvalStatus +from agenteval.storage.db import ( + IntelligentEvalDB, + IntelligentEvalDecisionLogDB, + OpenClawCronPoolDB, + utc_now, +) +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + IntelligentEvalDB, + IntelligentEvalSessionDB, + IntelligentEvalTaskQueueDB, + ) + + engine = create_engine( + f"sqlite:///{tmp_path / 'test.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + yield TestClient(app) + app.dependency_overrides.clear() + session.close() + engine.dispose() + + +@pytest.fixture() +def db_session(client): + return next(app.dependency_overrides[get_db]()) + + +# T3.1 — heartbeat updates last_active_at / status / current_eval_id + + +def test_heartbeat_updates_cron_fields(client: TestClient, db_session: Session): + cron = OpenClawCronPoolDB( + openclaw_cron_id="cron-1", + status="idle", + last_active_at=utc_now() - timedelta(hours=1), + ) + db_session.add(cron) + db_session.commit() + + response = client.post( + "/api/openclaw/crons/cron-1/heartbeat", + json={"status": "busy", "current_eval_id": "eval-42"}, + ) + assert response.status_code == 200 + + db_session.refresh(cron) + assert cron.status == "busy" + assert cron.current_eval_id == "eval-42" + + +def test_heartbeat_404_for_unknown_cron(client: TestClient): + response = client.post( + "/api/openclaw/crons/does-not-exist/heartbeat", + json={"status": "busy", "current_eval_id": None}, + ) + assert response.status_code == 404 + + +# T3.2 — decision-logs POST persists to DB + + +def test_create_decision_log_persists(client: TestClient, db_session: Session): + import uuid + eval_db = IntelligentEvalDB( + id=str(uuid.uuid4()), + name="eval-dl", + target_id="t1", + status=IntelligentEvalStatus.EXECUTING.value, + ) + db_session.add(eval_db) + db_session.commit() + + response = client.post( + f"/api/intelligent-evals/{eval_db.id}/decision-logs", + json={ + "decision_type": "execute_session", + "reason": "slot_due", + "context": {"slot": "8-10h", "deficit": 2}, + "cron_id": "cron-1", + }, + ) + assert response.status_code == 200 + + logs = db_session.exec( + select(IntelligentEvalDecisionLogDB).where( + IntelligentEvalDecisionLogDB.eval_id == eval_db.id + ) + ).all() + assert len(logs) == 1 + assert logs[0].decision_type == "execute_session" + assert logs[0].reason == "slot_due" + + +def test_create_decision_log_404_for_unknown_eval(client: TestClient): + import uuid + response = client.post( + f"/api/intelligent-evals/{uuid.uuid4()}/decision-logs", + json={ + "decision_type": "execute_session", + "reason": "slot_due", + "context": {}, + "cron_id": "cron-1", + }, + ) + assert response.status_code == 404 + + +# T3.3 — decision-logs GET lists logs + + +def test_list_decision_logs_returns_inserted(client: TestClient, db_session: Session): + import uuid + eval_db = IntelligentEvalDB( + id=str(uuid.uuid4()), + name="eval-dl-list", + target_id="t1", + status=IntelligentEvalStatus.EXECUTING.value, + ) + db_session.add(eval_db) + db_session.commit() + + for dtype in ("execute_session", "wait", "start_analysis"): + r = client.post( + f"/api/intelligent-evals/{eval_db.id}/decision-logs", + json={"decision_type": dtype, "reason": "test", "context": {}, "cron_id": "cron-1"}, + ) + assert r.status_code == 200 + + response = client.get(f"/api/intelligent-evals/{eval_db.id}/decision-logs") + assert response.status_code == 200 + data = response.json() + assert "logs" in data + types = {log["decision_type"] for log in data["logs"]} + assert types == {"execute_session", "wait", "start_analysis"}