diff --git a/.env.example b/.env.example index bfbe7cb..7edc5b4 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,11 @@ AGENTEVAL_OPENCLAW_AUTH_TOKEN=change-me-in-production # Optional override of the built frontend dist path (default: frontend/web/dist). # AGENTEVAL_FRONTEND_DIST_PATH=/app/frontend/web/dist +# ── Evaluation ───────────────────────────────────────────────── +# Seconds to wait for the target's reply per turn before recording no-reply. +# Raise this for slow targets (e.g. digital employees averaging near 30s). +AGENTEVAL_POLL_REPLY_TIMEOUT=30 + # ── Webhook ──────────────────────────────────────────────────── # If set, AgentEvalTool will POST run completion summaries to this URL. # Example: AGENTEVAL_WEBHOOK_URL=https://your-service.example.com/webhooks/agenteval diff --git a/backend/agenteval/config/settings.py b/backend/agenteval/config/settings.py index 7aa9b7c..032bdf1 100644 --- a/backend/agenteval/config/settings.py +++ b/backend/agenteval/config/settings.py @@ -76,6 +76,12 @@ class Settings(BaseSettings): description="Comma-separated list of allowed file extensions for upload.", ) + # ── Evaluation ───────────────────────────────────────────────── + poll_reply_timeout: float = Field( + default=30.0, + description="Seconds to wait for the target's reply per turn before recording no-reply.", + ) + # ── Webhook ──────────────────────────────────────────────────── webhook_url: Optional[str] = Field( default=None, diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index d2f1957..f54512c 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -12,6 +12,7 @@ from typing import Any, Callable, Optional from agenteval.channels.base import EvalChannel from agenteval.channels.factory import ChannelFactory +from agenteval.config import get_settings from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.rules import RuleResult, get_rule from agenteval.model_gateway import ModelGateway @@ -85,7 +86,9 @@ class EvalEngine: self.run_repo = run_repo or RunRepository(self.session) self.result_repo = result_repo or ResultRepository(self.session) self.cancel_token = cancel_token or asyncio.Event() - self.timeout_config = timeout_config or TimeoutConfig() + self.timeout_config = timeout_config or TimeoutConfig( + poll_reply=get_settings().poll_reply_timeout, + ) self.triggered_by = triggered_by self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases)) # Collects fatal case-level errors (e.g. dynamic message generation diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index c16b5ba..298daa2 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -39,3 +39,9 @@ def test_derived_falls_back_when_only_wildcard(): def test_derived_falls_back_when_empty(): s = Settings(allowed_origins=[]) assert s.openclaw_ws_origin == "http://localhost:8000" + + +def test_poll_reply_timeout_default_and_env(monkeypatch): + assert Settings().poll_reply_timeout == 30.0 + monkeypatch.setenv("AGENTEVAL_POLL_REPLY_TIMEOUT", "60") + assert Settings().poll_reply_timeout == 60.0