feat(engine): make poll_reply timeout configurable via env
Some checks failed
CI / test (push) Failing after 45s

被评数字员工响应普遍逼近 30s 硬编码轮询超时,越线的轮次被记为无回复
(run 427b14bb round 2 实测 31.4s 超时)。新增
AGENTEVAL_POLL_REPLY_TIMEOUT(默认 30s),engine 未显式传入
timeout_config 时从 settings 取值,慢目标可放宽。
This commit is contained in:
sinohqb 2026-07-30 09:58:08 +08:00
parent 5db0ede4f4
commit 1345daddd2
4 changed files with 21 additions and 1 deletions

View File

@ -37,6 +37,11 @@ AGENTEVAL_OPENCLAW_AUTH_TOKEN=change-me-in-production
# Optional override of the built frontend dist path (default: frontend/web/dist). # Optional override of the built frontend dist path (default: frontend/web/dist).
# AGENTEVAL_FRONTEND_DIST_PATH=/app/frontend/web/dist # AGENTEVAL_FRONTEND_DIST_PATH=/app/frontend/web/dist
# ── Evaluation ─────────────────────────────────────────────────
# Seconds to wait for the target's reply per turn before recording no-reply.
# Raise this for slow targets (e.g. digital employees averaging near 30s).
AGENTEVAL_POLL_REPLY_TIMEOUT=30
# ── Webhook ──────────────────────────────────────────────────── # ── Webhook ────────────────────────────────────────────────────
# If set, AgentEvalTool will POST run completion summaries to this URL. # If set, AgentEvalTool will POST run completion summaries to this URL.
# Example: AGENTEVAL_WEBHOOK_URL=https://your-service.example.com/webhooks/agenteval # Example: AGENTEVAL_WEBHOOK_URL=https://your-service.example.com/webhooks/agenteval

View File

@ -76,6 +76,12 @@ class Settings(BaseSettings):
description="Comma-separated list of allowed file extensions for upload.", description="Comma-separated list of allowed file extensions for upload.",
) )
# ── Evaluation ─────────────────────────────────────────────────
poll_reply_timeout: float = Field(
default=30.0,
description="Seconds to wait for the target's reply per turn before recording no-reply.",
)
# ── Webhook ──────────────────────────────────────────────────── # ── Webhook ────────────────────────────────────────────────────
webhook_url: Optional[str] = Field( webhook_url: Optional[str] = Field(
default=None, default=None,

View File

@ -12,6 +12,7 @@ from typing import Any, Callable, Optional
from agenteval.channels.base import EvalChannel from agenteval.channels.base import EvalChannel
from agenteval.channels.factory import ChannelFactory from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
from agenteval.evaluation.rules import RuleResult, get_rule from agenteval.evaluation.rules import RuleResult, get_rule
from agenteval.model_gateway import ModelGateway from agenteval.model_gateway import ModelGateway
@ -85,7 +86,9 @@ class EvalEngine:
self.run_repo = run_repo or RunRepository(self.session) self.run_repo = run_repo or RunRepository(self.session)
self.result_repo = result_repo or ResultRepository(self.session) self.result_repo = result_repo or ResultRepository(self.session)
self.cancel_token = cancel_token or asyncio.Event() self.cancel_token = cancel_token or asyncio.Event()
self.timeout_config = timeout_config or TimeoutConfig() self.timeout_config = timeout_config or TimeoutConfig(
poll_reply=get_settings().poll_reply_timeout,
)
self.triggered_by = triggered_by self.triggered_by = triggered_by
self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases)) self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases))
# Collects fatal case-level errors (e.g. dynamic message generation # Collects fatal case-level errors (e.g. dynamic message generation

View File

@ -39,3 +39,9 @@ def test_derived_falls_back_when_only_wildcard():
def test_derived_falls_back_when_empty(): def test_derived_falls_back_when_empty():
s = Settings(allowed_origins=[]) s = Settings(allowed_origins=[])
assert s.openclaw_ws_origin == "http://localhost:8000" assert s.openclaw_ws_origin == "http://localhost:8000"
def test_poll_reply_timeout_default_and_env(monkeypatch):
assert Settings().poll_reply_timeout == 30.0
monkeypatch.setenv("AGENTEVAL_POLL_REPLY_TIMEOUT", "60")
assert Settings().poll_reply_timeout == 60.0