初始化提交

This commit is contained in:
chenxin committed 2026-08-21 16:00:13 +08:00
commit 887c1f449e
963 files changed
+166869

No files matched your search

+965
View File
@@ -0,0 +1,965 @@
"""Agent 循环单元测试(流式版):标准 ReAct/Agent loop 的核心场景 + 累加器单测。
mock 边界:
- ``app.application.agent.runtime.ProviderManagerFactory.create`` 的 ``.chat_stream()`` —— 不真实调 LLM,
按轮次返回预置的 chunk 序列(async generator)。
- ``tool_runtime.execute`` —— 不真实调沙箱(用 AsyncMock 返回 ToolResult 或抛异常)。
"""
from __future__ import annotations
import asyncio
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.ai_provider import TokenUsage
from app.ai_provider.llm import ToolCallAccumulator
from app.ai_provider.types.chat import ChatChunk
from app.ai_provider.types.messages import (
AIMessage,
BaseMessage,
HumanMessage,
SystemMessage,
ToolMessage,
)
from app.application.agent import (
AgentContext,
AgentEvent,
AgentRunContext,
AgentTurnResult,
ExtensionRegistry,
run_agent_turn,
)
from app.application.agent.confirmation import ConfirmationRegistry
from app.application.agent.tools import default_tools, to_ai_provider_tools
from app.application.tool_runtime.builtins import builtin_specs
from app.modules.safety.exceptions import RequiresConfirmation
from app.modules.tool_runtime.definition import ToolResult
@pytest.fixture
def base_messages() -> list[BaseMessage]:
return [
SystemMessage(content="你是测试助手。"),
HumanMessage(content="请用 echo 工具回显 'hello'。"),
]
@pytest.fixture
def tools() -> list[dict[str, Any]]:
return default_tools()
# --------------------------------------------------------------------------- #
# 流式测试辅助:构造 chunk / chunk 序列 / 假 router / 假 tool_runtime
# --------------------------------------------------------------------------- #
def _chunk(
*,
content: str = "",
reasoning: str = "",
tool_call_chunks: list[dict[str, Any]] | None = None,
usage: TokenUsage | None = None,
finish_reason: str | None = None,
) -> ChatChunk:
"""构造单个 ChatChunk。"""
return ChatChunk(
delta_content=content,
delta_reasoning=reasoning,
finish_reason=finish_reason,
usage=usage,
tool_call_chunks=tool_call_chunks or [],
)
async def _chunk_iter(chunks: list[ChatChunk]):
"""把一组 chunk 包装成 async generator(模拟 router.stream 的返回值)。"""
for c in chunks:
yield c
def _split_tool_call_stream(
*,
index: int = 0,
call_id: str = "call_echo",
name: str = "echo",
args: dict[str, Any],
) -> list[dict[str, Any]]:
"""把一次 tool_call 拆成分片 chunk,模拟真实流式 tool_call_chunks:
第 1 片带 id + name + 空 args;后续把 args JSON 拆两段追加。最后附 finish_reason 片。
"""
args_json = json.dumps(args)
mid = max(1, len(args_json) // 2)
return [
{"index": index, "id": call_id, "name": name, "args": ""},
{"index": index, "args": args_json[:mid]},
{"index": index, "args": args_json[mid:]},
]
def _make_turn(
*,
content: str = "",
reasoning: str = "",
tool_calls: list[dict[str, Any]] | None = None,
usage: TokenUsage | None = None,
) -> list[ChatChunk]:
"""构造一轮流式的 chunk 序列。
若提供 ``tool_calls``(``[{"id":..., "name":..., "args": {...}}]``),则把每个
tool_call 拆成分片并附在序列末尾;否则只产出 content/reasoning chunk。
"""
chunks: list[ChatChunk] = []
if reasoning:
chunks.append(_chunk(reasoning=reasoning))
if content:
chunks.append(_chunk(content=content))
if tool_calls:
for idx, tc in enumerate(tool_calls):
chunks.append(
_chunk(
tool_call_chunks=_split_tool_call_stream(
index=idx,
call_id=tc.get("id", f"call_{tc['name']}"),
name=tc["name"],
args=tc.get("args", {}),
)
)
)
# usage 放在最后一个 chunk(与 OpenAI streaming 的 final chunk 习惯一致)
if chunks:
last = chunks[-1]
chunks[-1] = ChatChunk(
delta_content=last.delta_content,
delta_reasoning=last.delta_reasoning,
finish_reason="tool_calls" if tool_calls else "stop",
usage=usage or TokenUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
tool_call_chunks=last.tool_call_chunks,
)
else:
chunks.append(
_chunk(
usage=usage or TokenUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
finish_reason="stop",
)
)
return chunks
def _make_tool_result(stdout: str = "hello\n", ok: bool = True) -> ToolResult:
return ToolResult(
ok=ok,
exit_code=0 if ok else 1,
stdout=stdout,
stderr="",
duration_ms=3,
timeout=False,
truncated=False,
trace_id="t1",
)
def _patch_router_stream(
monkeypatch: pytest.MonkeyPatch, turns: list[list[ChatChunk]]
) -> MagicMock:
"""让 ``ProviderManagerFactory.create().chat_stream()`` 按顺序返回 turns(每轮 chunk 列表)。
``chat_stream`` 是 ``async def`` 返回 ``AsyncIterator[ChatChunk]``,故这里用 ``AsyncMock``
返回值设为对应 async generator。
"""
call_count = {"n": 0}
async def fake_chat_stream(_messages, **_kwargs): # noqa: ANN001
idx = call_count["n"]
call_count["n"] += 1
if idx >= len(turns):
raise AssertionError(f"chat_stream 被调用了 {idx + 1} 次,但只预期 {len(turns)} 次")
async for chunk in _chunk_iter(turns[idx]):
yield chunk
manager = MagicMock()
manager.chat_stream = fake_chat_stream
monkeypatch.setattr(
"app.application.agent.runtime.ProviderManagerFactory.create",
lambda: manager,
)
return manager
def _make_tool_runtime(execute_result: ToolResult | Exception) -> MagicMock:
"""构造一个 ToolRuntime 替身;execute 要么返回 ToolResult,要么抛给定异常。"""
runtime = MagicMock()
if isinstance(execute_result, Exception):
runtime.execute = AsyncMock(side_effect=execute_result)
else:
runtime.execute = AsyncMock(return_value=execute_result)
return runtime
# --------------------------------------------------------------------------- #
# 1. 正常闭环:第一轮流式 tool_call(分片),第二轮纯 content
# --------------------------------------------------------------------------- #
async def test_message_lifecycle_covers_assistant_updates_and_tool_results(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
_patch_router_stream(
monkeypatch,
[
_make_turn(
reasoning="先调用工具",
tool_calls=[{"id": "call_echo", "name": "echo", "args": {"text": "hi"}}],
),
_make_turn(content="完成"),
],
)
registry = ExtensionRegistry()
events: list[AgentEvent] = []
async def listen(event: AgentEvent, _run: AgentRunContext) -> None:
events.append(event)
registry.add_event_listener(listen)
await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=_make_tool_runtime(_make_tool_result()),
extensions=registry,
)
event_types = [event.type for event in events]
assert event_types.count("message.started") == 3
assert event_types.count("message.ended") == 3
assert "message.updated" in event_types
tool_ended = next(
event
for event in events
if event.type == "message.ended" and event.payload["role"] == "tool"
)
assert tool_ended.payload["message"]["tool_call_id"] == "call_echo"
async def test_normal_loop_completes_after_tool_call(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
reasoning="我需要用 echo 工具。",
tool_calls=[{"id": "call_echo", "name": "echo", "args": {"text": "hello"}}],
)
second = _make_turn(content="已为你回显:hello", reasoning="已收到工具结果,准备总结。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_tool_runtime(_make_tool_result(stdout="hello\n"))
# 收集 on_chunk / on_tool_event 调用
chunks_seen: list[ChatChunk] = []
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_chunk(chunk: ChatChunk) -> None:
chunks_seen.append(chunk)
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
model_hint="test-model",
on_chunk=on_chunk,
on_tool_event=on_tool_event,
)
assert isinstance(result, AgentTurnResult)
assert result.iterations == 2
assert result.truncated is False
assert result.content == "已为你回显:hello"
# reasoning 用最后一轮(最终回复轮)的累积值
assert result.reasoning == "已收到工具结果,准备总结。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["name"] == "echo"
assert result.tool_history[0]["status"] == "success"
assert result.tool_history[0]["args"] == {"text": "hello"}
assert "hello" in result.tool_history[0]["result"]
# 流式 chunk 被透传多次(至少两轮的 chunk 数)
assert len(chunks_seen) >= 2
# 工具事件:started + completed 各 1 次
assert len(tool_events) == 2
assert tool_events[0][0] == "tool.started"
assert tool_events[0][1]["tool_name"] == "echo"
assert tool_events[0][1]["tool_call_id"] == "call_echo"
assert tool_events[1][0] == "tool.completed"
assert tool_events[1][1]["status"] == "success"
assert "hello" in tool_events[1][1]["result"]
# messages 正确追加了 AIMessage(tool_calls) + ToolMessage + 最终 AIMessage
# base_messages 原长度为 2(SystemMessage + HumanMessage)
assert len(base_messages) == 5
ai_msg = base_messages[2]
assert isinstance(ai_msg, AIMessage)
assert ai_msg.tool_calls[0]["name"] == "echo"
tool_msg = base_messages[3]
assert isinstance(tool_msg, ToolMessage)
assert tool_msg.tool_call_id == "call_echo"
assert "hello" in tool_msg.content
final_msg = base_messages[4]
assert isinstance(final_msg, AIMessage)
assert final_msg.content == "已为你回显:hello"
# usage 累计两轮
assert result.usage.total_tokens == 30
# tool_runtime.execute 被调用了一次,参数正确
tool_runtime.execute.assert_awaited_once()
call_kwargs = tool_runtime.execute.call_args
assert call_kwargs.args[0] == "echo"
assert call_kwargs.args[1] == {"text": "hello"}
# --------------------------------------------------------------------------- #
# 2. 工具失败不中断:execute 抛异常,stream 第二轮仍能正常返回 content
# --------------------------------------------------------------------------- #
async def test_tool_failure_does_not_break_loop(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_echo", "name": "echo", "args": {"text": "hi"}}],
)
second = _make_turn(content="工具坏了,但我还是回复你。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_tool_runtime(RuntimeError("sandbox down"))
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=on_tool_event,
)
assert result.iterations == 2
assert result.truncated is False
assert result.content == "工具坏了,但我还是回复你。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["status"] == "error"
assert "sandbox down" in result.tool_history[0]["result"]
# 错误信息作为 ToolMessage 回填
tool_msg = base_messages[3]
assert isinstance(tool_msg, ToolMessage)
assert "sandbox down" in tool_msg.content
# 工具事件 completed 带 status=error
assert [e[0] for e in tool_events] == ["tool.started", "tool.completed"]
assert tool_events[1][1]["status"] == "error"
assert "sandbox down" in tool_events[1][1]["result"]
# --------------------------------------------------------------------------- #
# 3. 上限截断:stream 每次都返回 tool_call,max_iterations=3
# --------------------------------------------------------------------------- #
async def test_loop_truncates_at_max_iterations(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
# 每次都返回 tool_call,让循环无法终止
looping_turn = _make_turn(
tool_calls=[{"id": "call_echo", "name": "echo", "args": {"text": "loop"}}],
)
_patch_router_stream(monkeypatch, [looping_turn, looping_turn, looping_turn])
tool_runtime = _make_tool_runtime(_make_tool_result(stdout="loop\n"))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
max_iterations=3,
)
assert result.iterations == 3
assert result.truncated is True
assert len(result.tool_history) == 3
assert all(h["status"] == "success" for h in result.tool_history)
# messages 累积:base 2 + 3 轮 × (AIMessage + ToolMessage) = 8
assert len(base_messages) == 8
# --------------------------------------------------------------------------- #
# 4. 无工具直接返回:stream 第一次就是纯 content
# --------------------------------------------------------------------------- #
async def test_no_tool_call_returns_immediately(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
direct = _make_turn(content="直接回答,不需要工具。")
_patch_router_stream(monkeypatch, [direct])
tool_runtime = _make_tool_runtime(_make_tool_result())
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=on_tool_event,
)
assert result.iterations == 1
assert result.truncated is False
assert result.content == "直接回答,不需要工具。"
assert result.tool_history == []
# messages 只追加了一条 AIMessage
assert len(base_messages) == 3
assert isinstance(base_messages[2], AIMessage)
# execute 从未被调用
tool_runtime.execute.assert_not_awaited()
# 无工具事件
assert tool_events == []
# --------------------------------------------------------------------------- #
# 5. 工具转换辅助:to_ai_provider_tools / default_tools 基本正确
# --------------------------------------------------------------------------- #
def test_to_ai_provider_tools_produces_openai_function_schema() -> None:
specs = list(builtin_specs().values())
tools = to_ai_provider_tools(specs)
assert len(tools) == len(specs)
for tool, spec in zip(tools, specs, strict=True):
assert tool["type"] == "function"
fn = tool["function"]
assert fn["name"] == spec.name
assert fn["description"] == spec.description
assert fn["parameters"]["type"] == "object"
def test_default_tools_returns_core_only_by_default() -> None:
"""default_tools 默认只返回 5 个核心沙箱工具(不含联网工具)。"""
tools = default_tools()
names = {t["function"]["name"] for t in tools}
assert names == {"datetime", "read_file", "list_dir", "write_file", "shell"}
assert len(tools) == 5
def test_default_tools_returns_core_only() -> None:
"""default_tools 返回 5 个核心沙箱工具。"""
tools = default_tools()
names = {t["function"]["name"] for t in tools}
assert names == {"datetime", "read_file", "list_dir", "write_file", "shell"}
assert len(tools) == 5
def test_run_agent_turn_rejects_non_positive_max_iterations(
base_messages: list[BaseMessage], tools: list[dict[str, Any]]
) -> None:
tool_runtime = _make_tool_runtime(_make_tool_result())
with pytest.raises(ValueError):
import asyncio
asyncio.get_event_loop().run_until_complete(
run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
max_iterations=0,
)
)
# --------------------------------------------------------------------------- #
# 6. ToolCallAccumulator 单元测试
# --------------------------------------------------------------------------- #
def test_accumulator_empty_drain_returns_empty_list() -> None:
acc = ToolCallAccumulator()
drained = acc.drain()
assert drained == []
def test_accumulator_assembles_fragmented_args_across_chunks() -> None:
"""args 分片到达也能拼回完整 JSON。"""
acc = ToolCallAccumulator()
# 片段:先到 id+name,args 分 "hello"、""、" world" 三段
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "id": "c1", "name": "echo", "args": ""}]))
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "args": "hello"}]))
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "args": ""}]))
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "args": " world"}]))
drained = acc.drain()
assert len(drained) == 1
entry = drained[0]
assert entry["index"] == 0
assert entry["id"] == "c1"
assert entry["name"] == "echo"
assert entry["args"] == "hello world"
def test_accumulator_parallel_multiple_tool_calls() -> None:
"""index 0 和 1 的 tool_call 并行累积。"""
acc = ToolCallAccumulator()
acc.add_chunk(
_chunk(
tool_call_chunks=[
{"index": 0, "id": "c1", "name": "echo", "args": '{"text"'},
{"index": 1, "id": "c2", "name": "datetime", "args": "{}"},
]
)
)
acc.add_chunk(
_chunk(
tool_call_chunks=[
{"index": 0, "args": ': "a"}'},
]
)
)
drained = acc.drain()
assert len(drained) == 2
assert drained[0]["index"] == 0
assert drained[0]["name"] == "echo"
assert drained[0]["args"] == '{"text": "a"}'
assert drained[1]["index"] == 1
assert drained[1]["name"] == "datetime"
assert drained[1]["args"] == "{}"
def test_accumulator_args_arrive_before_name() -> None:
"""分片中先到 args 片段、后到 name,也能正确合并。"""
acc = ToolCallAccumulator()
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "args": '{"x": '}]))
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "id": "c1", "name": "calc", "args": "1"}]))
acc.add_chunk(_chunk(tool_call_chunks=[{"index": 0, "args": "}"}]))
drained = acc.drain()
assert len(drained) == 1
entry = drained[0]
assert entry["id"] == "c1"
assert entry["name"] == "calc"
assert entry["args"] == '{"x": 1}'
def test_accumulator_ignores_chunks_without_tool_call_chunks() -> None:
"""非 tool_call chunk(只有 content)不会影响累加器状态。"""
acc = ToolCallAccumulator()
acc.add_chunk(_chunk(content="hello"))
acc.add_chunk(_chunk(reasoning="thinking"))
acc.add_chunk(_chunk())
drained = acc.drain()
assert drained == []
# --------------------------------------------------------------------------- #
# 7. 工具确认挂起/恢复(阶段3 步骤1)
# --------------------------------------------------------------------------- #
def _make_confirmation_tool_runtime(
*, exc: Exception | None = None, result: ToolResult | None = None
) -> MagicMock:
"""构造 ToolRuntime:先抛 RequiresConfirmation(或给定 exc),再返回 result。
用于"批准后重试 confirmed=True 成功"的场景。result 缺省为成功 ToolResult。
"""
result = result or _make_tool_result(stdout="done\n")
if exc is not None:
runtime = MagicMock()
runtime.execute = AsyncMock(side_effect=[exc, result])
return runtime
raise ValueError("必须提供 exc")
def _make_deny_only_tool_runtime(exc: Exception) -> MagicMock:
"""构造 ToolRuntime:只抛一次 RequiresConfirmation(拒绝/超时不重试)。"""
runtime = MagicMock()
runtime.execute = AsyncMock(side_effect=exc)
return runtime
async def test_context_transformer_runs_for_every_model_request_without_persisting(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_echo", "name": "echo", "args": {"text": "x"}}]
)
second = _make_turn(content="完成")
manager = _patch_router_stream(monkeypatch, [first, second])
runtime = MagicMock()
runtime.execute = AsyncMock(return_value=_make_tool_result())
registry = ExtensionRegistry()
transformed_calls: list[int] = []
async def transform(context: AgentContext, _run: AgentRunContext) -> AgentContext:
transformed_calls.append(len(context.messages))
return AgentContext(
(*context.messages, HumanMessage(content="临时注入")),
context.tools,
context.model_hint,
)
registry.add_context_transformer(transform)
await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=runtime,
extensions=registry,
)
assert transformed_calls == [2, 4]
assert all(message.content != "临时注入" for message in base_messages)
assert manager.chat_stream is not None
async def test_max_tokens_marks_agent_result_truncated(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
turn = _make_turn(content="未完成")
turn[-1].finish_reason = "length"
_patch_router_stream(monkeypatch, [turn])
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=MagicMock(),
)
assert result.content == "未完成"
assert result.truncated is True
# --------------------------------------------------------------------------- #
# 7.1 确认 + 批准恢复:execute 第一次抛 RequiresConfirmation → 发事件 →
# resolve(cid, True) → confirmed=True 重试成功 → 第二轮纯 content
# --------------------------------------------------------------------------- #
async def test_confirmation_approved_resumes_and_succeeds(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_write", "name": "write_file", "args": {"path": "a.txt"}}],
)
second = _make_turn(content="已写入文件。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_confirmation_tool_runtime(
exc=RequiresConfirmation("策略 ask 需用户确认", risk_level="L2")
)
registry = ConfirmationRegistry()
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
# 收到 requires_confirmation 事件后,手动触发批准恢复循环
if event_type == "tool.requires_confirmation":
cid = payload["confirmation_id"]
asyncio.create_task(registry.resolve(cid, True))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=on_tool_event,
confirmation_registry=registry,
run_id="test-run-1",
character_id="test-char-1",
)
assert result.iterations == 2
assert result.content == "已写入文件。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["status"] == "success"
# 事件顺序:started(初次) → requires_confirmation → started(重试,带 confirmed=True)
# → completed(success)
types = [e[0] for e in tool_events]
assert types == [
"tool.started",
"tool.requires_confirmation",
"tool.started",
"tool.completed",
]
req_event = tool_events[1][1]
assert req_event["tool_name"] == "write_file"
assert req_event["risk_level"] == "L2"
assert req_event["confirmation_id"].startswith("confirm_")
# 重试时带 confirmed=True 标记
assert tool_events[2][1].get("confirmed") is True
assert tool_events[3][1]["status"] == "success"
# execute 被调用 2 次:第一次 confirmed=False(默认),第二次 confirmed=True
assert tool_runtime.execute.await_count == 2
first_call = tool_runtime.execute.call_args_list[0]
second_call = tool_runtime.execute.call_args_list[1]
assert first_call.kwargs["confirmed"] is False
assert second_call.kwargs["confirmed"] is True
# registry 已清理(resolve 后 Future 出列)
assert not registry.has(req_event["confirmation_id"])
# --------------------------------------------------------------------------- #
# 7.2 确认 + 拒绝:resolve(cid, False) → 回填"用户拒绝" → 循环继续
# --------------------------------------------------------------------------- #
async def test_confirmation_denied_backfills_and_continues(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_shell", "name": "shell", "args": {"command": "rm -rf /"}}],
)
second = _make_turn(content="好的,我不执行了。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_deny_only_tool_runtime(
RequiresConfirmation("策略 always_ask 需用户确认", risk_level="L3")
)
registry = ConfirmationRegistry()
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
if event_type == "tool.requires_confirmation":
cid = payload["confirmation_id"]
asyncio.create_task(registry.resolve(cid, False))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=on_tool_event,
confirmation_registry=registry,
run_id="test-run-2",
character_id="test-char-2",
confirmation_timeout=1.0,
)
assert result.iterations == 2
assert result.content == "好的,我不执行了。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["status"] == "denied"
assert "用户拒绝" in result.tool_history[0]["result"]
# ToolMessage 回填的是"用户拒绝"
tool_msg = base_messages[3]
assert isinstance(tool_msg, ToolMessage)
assert "用户拒绝" in tool_msg.content
# 事件:started → requires_confirmation → completed(status=denied)
types = [e[0] for e in tool_events]
assert types == ["tool.started", "tool.requires_confirmation", "tool.completed"]
assert tool_events[-1][1]["status"] == "denied"
# execute 只被调用 1 次(拒绝不重试)
assert tool_runtime.execute.await_count == 1
# --------------------------------------------------------------------------- #
# 7.3 确认超时按拒绝:confirmation_timeout 极短,不 resolve → 超时 → 回填 → 继续
# --------------------------------------------------------------------------- #
async def test_confirmation_timeout_treated_as_denied(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_write", "name": "write_file", "args": {"path": "x.txt"}}],
)
second = _make_turn(content="收到,跳过写入。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_deny_only_tool_runtime(RequiresConfirmation("需确认", risk_level="L2"))
registry = ConfirmationRegistry()
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=None, # 不需要事件断言
confirmation_registry=registry,
confirmation_timeout=1.5, # > poll_interval(0.5s)确保至少完成一次轮询
)
assert result.iterations == 2
assert result.content == "收到,跳过写入。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["status"] == "denied"
assert "超时" in result.tool_history[0]["result"]
# execute 只被调用 1 次(超时不重试)
assert tool_runtime.execute.await_count == 1
# --------------------------------------------------------------------------- #
# 7.4 registry=None 降级:execute 抛 RequiresConfirmation → 回填"未启用" → 继续
# --------------------------------------------------------------------------- #
async def test_confirmation_degraded_when_registry_none(
monkeypatch: pytest.MonkeyPatch,
base_messages: list[BaseMessage],
tools: list[dict[str, Any]],
) -> None:
first = _make_turn(
tool_calls=[{"id": "call_shell", "name": "shell", "args": {"command": "ls"}}],
)
second = _make_turn(content="好的,跳过了。")
_patch_router_stream(monkeypatch, [first, second])
tool_runtime = _make_deny_only_tool_runtime(RequiresConfirmation("需确认", risk_level="L3"))
tool_events: list[tuple[str, dict[str, Any]]] = []
async def on_tool_event(event_type: str, payload: dict[str, Any]) -> None:
tool_events.append((event_type, payload))
result = await run_agent_turn(
messages=base_messages,
tools=tools,
tool_runtime=tool_runtime,
on_tool_event=on_tool_event,
# 不传 confirmation_registry → 降级路径
)
assert result.iterations == 2
assert result.content == "好的,跳过了。"
assert len(result.tool_history) == 1
assert result.tool_history[0]["status"] == "skipped"
assert "需要用户确认" in result.tool_history[0]["result"]
assert "未启用" in result.tool_history[0]["result"]
# 事件:started → completed(status=skipped)(不发 requires_confirmation)
types = [e[0] for e in tool_events]
assert types == ["tool.started", "tool.completed"]
assert tool_events[-1][1]["status"] == "skipped"
# execute 只被调用 1 次
assert tool_runtime.execute.await_count == 1
# --------------------------------------------------------------------------- #
# 7.5 ConfirmationRegistry 单元测试
# --------------------------------------------------------------------------- #
async def test_confirmation_registry_resolve_returns_false_for_unknown() -> None:
registry = ConfirmationRegistry()
assert await registry.resolve("unknown_id", True) is False
assert registry.has("unknown_id") is False
async def test_confirmation_registry_resolves_only_requested_confirmation() -> None:
from app.db.session import AsyncSessionLocal
from app.models.safety import PendingConfirmationRecord
registry = ConfirmationRegistry()
async with AsyncSessionLocal() as session:
session.add_all(
[
PendingConfirmationRecord(
id="confirm_first",
run_id="run_shared",
tool="first",
input_json="{}",
risk_level="L2",
status="pending",
),
PendingConfirmationRecord(
id="confirm_second",
run_id="run_shared",
tool="second",
input_json="{}",
risk_level="L2",
status="pending",
),
]
)
await session.commit()
assert await registry.resolve(
"run_shared", True, confirmation_id="confirm_second"
) is True
async with AsyncSessionLocal() as session:
first = await session.get(PendingConfirmationRecord, "confirm_first")
second = await session.get(PendingConfirmationRecord, "confirm_second")
assert first is not None and first.status == "pending"
assert second is not None and second.status == "approved"
async def test_confirmation_registry_resolve_returns_false_after_done() -> None:
from app.db.session import AsyncSessionLocal
from app.models.safety import PendingConfirmationRecord
registry = ConfirmationRegistry()
assert registry.has("cid_1") is False
# 创建 PendingConfirmationRecord(wait/resolve 的 DB 前提)
async with AsyncSessionLocal() as session:
session.add(
PendingConfirmationRecord(
id="rec_cid_1",
run_id="cid_1",
tool="test",
input_json="{}",
risk_level="L2",
status="pending",
)
)
await session.commit()
running = asyncio.create_task(registry.wait("cid_1", poll_interval=0.1, timeout=0.5))
await asyncio.sleep(0.1)
assert registry.has("cid_1") is True
resolved = await registry.resolve("cid_1", True)
assert resolved is True
result = await running
assert result is True
# 已处理,再次 resolve 返回 False
assert await registry.resolve("cid_1", True) is False
assert registry.has("cid_1") is False