feat(engine): make poll_reply timeout configurable via env
Some checks failed
CI / test (push) Failing after 45s
Some checks failed
CI / test (push) Failing after 45s
被评数字员工响应普遍逼近 30s 硬编码轮询超时,越线的轮次被记为无回复 (run 427b14bb round 2 实测 31.4s 超时)。新增 AGENTEVAL_POLL_REPLY_TIMEOUT(默认 30s),engine 未显式传入 timeout_config 时从 settings 取值,慢目标可放宽。
This commit is contained in:
parent
5db0ede4f4
commit
1345daddd2
@ -37,6 +37,11 @@ AGENTEVAL_OPENCLAW_AUTH_TOKEN=change-me-in-production
|
|||||||
# Optional override of the built frontend dist path (default: frontend/web/dist).
|
# Optional override of the built frontend dist path (default: frontend/web/dist).
|
||||||
# AGENTEVAL_FRONTEND_DIST_PATH=/app/frontend/web/dist
|
# AGENTEVAL_FRONTEND_DIST_PATH=/app/frontend/web/dist
|
||||||
|
|
||||||
|
# ── Evaluation ─────────────────────────────────────────────────
|
||||||
|
# Seconds to wait for the target's reply per turn before recording no-reply.
|
||||||
|
# Raise this for slow targets (e.g. digital employees averaging near 30s).
|
||||||
|
AGENTEVAL_POLL_REPLY_TIMEOUT=30
|
||||||
|
|
||||||
# ── Webhook ────────────────────────────────────────────────────
|
# ── Webhook ────────────────────────────────────────────────────
|
||||||
# If set, AgentEvalTool will POST run completion summaries to this URL.
|
# If set, AgentEvalTool will POST run completion summaries to this URL.
|
||||||
# Example: AGENTEVAL_WEBHOOK_URL=https://your-service.example.com/webhooks/agenteval
|
# Example: AGENTEVAL_WEBHOOK_URL=https://your-service.example.com/webhooks/agenteval
|
||||||
|
|||||||
@ -76,6 +76,12 @@ class Settings(BaseSettings):
|
|||||||
description="Comma-separated list of allowed file extensions for upload.",
|
description="Comma-separated list of allowed file extensions for upload.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ── Evaluation ─────────────────────────────────────────────────
|
||||||
|
poll_reply_timeout: float = Field(
|
||||||
|
default=30.0,
|
||||||
|
description="Seconds to wait for the target's reply per turn before recording no-reply.",
|
||||||
|
)
|
||||||
|
|
||||||
# ── Webhook ────────────────────────────────────────────────────
|
# ── Webhook ────────────────────────────────────────────────────
|
||||||
webhook_url: Optional[str] = Field(
|
webhook_url: Optional[str] = Field(
|
||||||
default=None,
|
default=None,
|
||||||
|
|||||||
@ -12,6 +12,7 @@ from typing import Any, Callable, Optional
|
|||||||
|
|
||||||
from agenteval.channels.base import EvalChannel
|
from agenteval.channels.base import EvalChannel
|
||||||
from agenteval.channels.factory import ChannelFactory
|
from agenteval.channels.factory import ChannelFactory
|
||||||
|
from agenteval.config import get_settings
|
||||||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||||||
from agenteval.evaluation.rules import RuleResult, get_rule
|
from agenteval.evaluation.rules import RuleResult, get_rule
|
||||||
from agenteval.model_gateway import ModelGateway
|
from agenteval.model_gateway import ModelGateway
|
||||||
@ -85,7 +86,9 @@ class EvalEngine:
|
|||||||
self.run_repo = run_repo or RunRepository(self.session)
|
self.run_repo = run_repo or RunRepository(self.session)
|
||||||
self.result_repo = result_repo or ResultRepository(self.session)
|
self.result_repo = result_repo or ResultRepository(self.session)
|
||||||
self.cancel_token = cancel_token or asyncio.Event()
|
self.cancel_token = cancel_token or asyncio.Event()
|
||||||
self.timeout_config = timeout_config or TimeoutConfig()
|
self.timeout_config = timeout_config or TimeoutConfig(
|
||||||
|
poll_reply=get_settings().poll_reply_timeout,
|
||||||
|
)
|
||||||
self.triggered_by = triggered_by
|
self.triggered_by = triggered_by
|
||||||
self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases))
|
self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases))
|
||||||
# Collects fatal case-level errors (e.g. dynamic message generation
|
# Collects fatal case-level errors (e.g. dynamic message generation
|
||||||
|
|||||||
@ -39,3 +39,9 @@ def test_derived_falls_back_when_only_wildcard():
|
|||||||
def test_derived_falls_back_when_empty():
|
def test_derived_falls_back_when_empty():
|
||||||
s = Settings(allowed_origins=[])
|
s = Settings(allowed_origins=[])
|
||||||
assert s.openclaw_ws_origin == "http://localhost:8000"
|
assert s.openclaw_ws_origin == "http://localhost:8000"
|
||||||
|
|
||||||
|
|
||||||
|
def test_poll_reply_timeout_default_and_env(monkeypatch):
|
||||||
|
assert Settings().poll_reply_timeout == 30.0
|
||||||
|
monkeypatch.setenv("AGENTEVAL_POLL_REPLY_TIMEOUT", "60")
|
||||||
|
assert Settings().poll_reply_timeout == 60.0
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user