feat(review): agent 侧虚拟工具 inspect_region —— 按可读分辨率放大指定区域

解决三个缺口:① 放大没复用标定/瓦片数学(LLM 一次 zoom 只出一张,宽区域仍读不清);
②「看清没」由 VLM 自评(易过度自信/幻觉);③ 只能按 handle / 手算 bbox 指认,
图上看到但没 handle 的东西指认不了。

核心思路:把「放大」升级成一个语义化工具——LLM 只指认「要看哪里」,由 agent 复用
tiling.py 的标定数学算出该切几张、每张覆盖多大,用构造保证可读。

新增 dwg_review/inspect.py(agent 侧,不编 DLL):
- TOOL_SCHEMA:inspect_region(target, purpose);target 三形态 {handle} / {window} /
  {label,cx,cy,span},purpose = text(8px) / line(6px)。
- InspectState:单图框内「截图标签 → 世界可见范围」登记表,支持逐级下钻
  (返回的 inspect-N 可继续作为 label 指认)。
- resolve_target / _point_box:label+cx+cy → WCS。cy 是图像坐标(上→下),
  世界 y 必须翻转(y = y2 - cy*H)——单测专门钉住。
- plan_tiles:复用 world_per_px + tiles_for_box + subdivide;超 INSPECT_MAX_TILES(4)
  则硬失败并给出「缩小 target / 分多次请求」的指引(不静默降级);拿不到比例/像素
  则退回单张并如实告知。

关键几何:CAD 的 _.ZOOM _W 是等比适配(不拉伸),故「截图上的归一化坐标」必须按
max(W/ew, H/eh) 反算真实可见范围(新增 tiling.visible_box)——直接用送进去的窗口会偏。
zoom_to_window 返回的 view 是「加过 margin 的窗口」,不是可见范围,不能混用。

证据层配套:evidence_model.make_image(..., window=) 记录每张图的可见世界范围;
capture_images 用 view_px(overall 图原始像素)+ visible_box 逐张登记。

子循环接入:_run_readonly_tool 在只读白名单校验之前拦截 inspect_region(它不在
READ_ONLY_TOOLS 里,内部只调白名单内的只读 MCP 工具);_review_tools 启用时用
inspect_region 取代三个裸视口工具;inspect_region 计入 _VIEW_TOOLS → 结束复位视口。
配置:INSPECT_ENABLED(True) / INSPECT_MAX_TILES(4)。

端到端实测(真实图纸 1 图框):取证与审查两阶段均在最大化态、结束精确还原(4 项 PASS);
LLM 的 11 次取证调用里 2 次是 inspect_region,且都显式给 window + purpose:"text"
(x[20000,115000] y[-6500,3000],即 A0 图框底部横条=标题栏所在)。
验收口径达成:报告报出标题栏乱码 ???????????????,而 get_text_content 对全图返回 0 条文字
(本图文字全在块/自定义实体内)→ 块内文字是被视觉放大读出来的。
pytest 343 passed。
This commit is contained in:
gjm committed 2026-10-16 19:15:32 +08:00
1 parent 3bc64b6c3e
commit b757892c67
18 files changed
+1003 -45

No files matched your search

+8 -5
View File
@@ -65,12 +65,15 @@ def test_build_system_prompt_tool_hint_toggle():
assert "只读" not in without
def test_build_system_prompt_instructs_zoom_when_unreadable():
def test_build_system_prompt_instructs_inspect_region():
prompt = build_system_prompt(DEFAULT_CHECKLIST, allow_tools=True)
assert "看不清" in prompt
assert "zoom_to_window" in prompt
assert "zoom_to_entity" in prompt
assert "get_viewport_screenshot" in prompt
# 放大取证走 inspect_region 这一个语义化入口(不再让 LLM 自己组合裸视口工具)
assert "inspect_region" in prompt
assert "handle" in prompt and "label" in prompt and "window" in prompt
assert "cx" in prompt and "cy" in prompt
assert "从上到下" in prompt # cy 方向必须写清(图像坐标)
assert "inspect-N" in prompt # 返回标签可继续下钻
assert "不需要判断截图够不够清楚" in prompt
def test_build_system_prompt_requires_structured_json_output():
+6
View File
@@ -106,6 +106,12 @@ def test_review_obs_images_default_is_sane():
assert config.REVIEW_OBS_IMAGES >= 3
def test_inspect_defaults_are_sane():
# agent 侧虚拟工具默认启用;单次瓦片上限为正
assert config.INSPECT_ENABLED is True
assert config.INSPECT_MAX_TILES >= 1
def test_drawing_area_constants_have_expected_types():
assert isinstance(config.DRAWING_AREA_MAXIMIZE, bool)
assert isinstance(config.DRAWING_AREA_ENTER_WAIT, float)
+9 -1
View File
@@ -77,7 +77,15 @@ def test_collect_frame_assembles_evidence():
assert ev["blocks"]["total"] == 0
assert ev["polylines"]["total"] == 3
assert ev["source_type"] == "cad"
assert ev["images"] == [{"b64": "QUJD", "mime": "image/png", "label": "overall"}]
# 截图登记「实际可见世界范围」:拿不到绘图区像素时退回「窗口外扩 margin」的近似值
assert ev["images"] == [
{
"b64": "QUJD",
"mime": "image/png",
"label": "overall",
"window": (-2.0, -1.0, 102.0, 51.0),
}
]
def test_collect_frame_calls_expected_tools():
+1 -1
View File
@@ -52,7 +52,7 @@ def test_evidence_images_filters_empty_b64():
def test_evidence_images_legacy_fallback():
ev = {"image_b64": "LEG", "image_mime": "image/jpeg"}
assert em.evidence_images(ev) == [
{"b64": "LEG", "mime": "image/jpeg", "label": None}
{"b64": "LEG", "mime": "image/jpeg", "label": None, "window": None}
]
+348
View File
@@ -0,0 +1,348 @@
"""`dwg_review.inspect` 单元测试(agent 侧虚拟工具 `inspect_region`)。"""
import base64
import io
import pytest
from dwg_review import config, inspect
from dwg_review.inspect import (
InspectState,
_point_box,
inspect_region,
plan_tiles,
readable_px,
resolve_target,
)
from dwg_review.phases.review import review_frame
from tests.fakes import (
FakeLlmClient,
FakeMcpClient,
frame,
llm_calls,
llm_text,
make_ctx,
make_evidence,
ok_image,
)
@pytest.fixture
def inspect_env(monkeypatch):
"""钉死 inspect 相关配置,使用例不受外部环境变量影响。"""
for name, value in {
"TEXT_HEIGHT_MM": 2.0,
"TEXT_READABLE_PX": 8,
"LINE_READABLE_PX": 6,
"INSPECT_MAX_TILES": 4,
"INSPECT_ENABLED": True,
"TILE_OVERLAP": 0.0,
"IMAGE_MAX_EDGE": 1280,
"RENDER_WAIT": 0.0,
}.items():
monkeypatch.setattr(config, name, value)
return config
def _png_b64(w=2000, h=1414):
from PIL import Image
buf = io.BytesIO()
Image.new("RGB", (w, h), "black").save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
def _evidence(images=None, entities=None, sheet_scale=100.0, sheet_size="A0"):
"""图框 (0,0)-(120000,40000)、1:100、A0 的 evidence(含 overall 截图)。"""
ev = make_evidence()
ev["frame"] = frame([0, 0, 120000, 40000], sheet_size=sheet_size, sheet_scale=sheet_scale)
ev["window"] = {"min_x": 0.0, "min_y": 0.0, "max_x": 120000.0, "max_y": 40000.0}
ev["entities"] = entities or {
"total": 1,
"items": [{"handle": "A1", "type": "LINE", "bbox": [0, 0, 1000, 500]}],
}
if images is None:
images = [
{
"b64": _png_b64(),
"mime": "image/png",
"label": "overall",
"window": (0.0, 0.0, 120000.0, 40000.0),
}
]
ev["images"] = images
return ev
def _client(b64=None):
return FakeMcpClient(
responses={"get_viewport_screenshot": ok_image(b64 or _png_b64())}
)
# ---------------------------------------------------------------------------
# InspectState
# ---------------------------------------------------------------------------
def test_inspect_state_registers_evidence_labels_with_window():
ev = _evidence(images=[
{"b64": "A", "mime": "image/png", "label": "overall", "window": (0, 0, 10, 10)},
{"b64": "B", "mime": "image/png", "label": "tile-1", "window": [0, 0, 5, 5]},
{"b64": "C", "mime": "image/png", "label": "tile-2"}, # 无 window → 不登记
])
state = InspectState(ev)
assert set(state.labels) == {"overall", "tile-1"}
assert state.labels["tile-1"] == (0.0, 0.0, 5.0, 5.0)
def test_inspect_state_register_assigns_incrementing_labels():
state = InspectState(_evidence())
assert state.register((1, 2, 3, 4)) == "inspect-1"
assert state.register((0, 0, 1, 1)) == "inspect-2"
assert state.labels["inspect-1"] == (1, 2, 3, 4)
# ---------------------------------------------------------------------------
# _point_box / resolve_target
# ---------------------------------------------------------------------------
def test_point_box_flips_y_because_cy_is_image_down():
base = (0.0, 0.0, 100.0, 50.0)
top = _point_box(base, 0.5, 0.0, 0.5) # 图像顶部
bottom = _point_box(base, 0.5, 1.0, 0.5) # 图像底部
assert (top[1] + top[3]) / 2 == pytest.approx(50.0) # → 世界 y 大的一侧
assert (bottom[1] + bottom[3]) / 2 == pytest.approx(0.0)
def test_point_box_centers_and_clamps_span():
base = (0.0, 0.0, 100.0, 50.0)
box = _point_box(base, 0.5, 0.5, 0.5)
assert box == pytest.approx((25.0, 12.5, 75.0, 37.5))
assert _point_box(base, 0.5, 0.5, 5.0) == pytest.approx(base) # span 上限 1
tiny = _point_box(base, 0.5, 0.5, 0.0)
assert (tiny[2] - tiny[0]) == pytest.approx(100.0 * 0.02) # span 下限 0.02
def test_resolve_target_by_handle_is_case_insensitive():
ev = _evidence()
box, note = resolve_target(ev, {"handle": "a1"}, InspectState(ev))
assert box == (0.0, 0.0, 1000.0, 500.0)
assert "a1" in note
def test_resolve_target_handle_not_found():
ev = _evidence()
box, note = resolve_target(ev, {"handle": "ZZ"}, InspectState(ev))
assert box is None
assert "ZZ" in note and "window" in note
def test_resolve_target_by_window():
ev = _evidence()
box, note = resolve_target(
ev, {"window": {"min_x": 1, "min_y": 2, "max_x": 3, "max_y": 4}}, InspectState(ev)
)
assert box == (1.0, 2.0, 3.0, 4.0)
assert note
def test_resolve_target_bad_window():
ev = _evidence()
box, note = resolve_target(ev, {"window": {"min_x": 1}}, InspectState(ev))
assert box is None
assert "window 非法" in note
def test_resolve_target_by_label_point():
ev = _evidence()
state = InspectState(ev)
box, note = resolve_target(
ev, {"label": "overall", "cx": 0.5, "cy": 0.5, "span": 0.5}, state
)
# overall 覆盖 (0,0)-(120000,40000) → 中心 (60000, 20000),跨度为半幅
assert box == pytest.approx((30000.0, 10000.0, 90000.0, 30000.0))
assert "overall" in note
def test_resolve_target_unknown_label_lists_known():
ev = _evidence()
box, note = resolve_target(ev, {"label": "tile-9"}, InspectState(ev))
assert box is None
assert "tile-9" in note and "overall" in note
def test_resolve_target_rejects_bad_inputs():
ev = _evidence()
state = InspectState(ev)
assert resolve_target(ev, "nope", state)[0] is None
assert resolve_target(ev, {}, state)[0] is None
assert resolve_target(ev, {"label": "overall", "cx": "x"}, state)[0] is None
# ---------------------------------------------------------------------------
# plan_tiles
# ---------------------------------------------------------------------------
def test_readable_px_by_purpose(inspect_env):
assert readable_px("text") == inspect_env.TEXT_READABLE_PX
assert readable_px("line") == inspect_env.LINE_READABLE_PX
def test_plan_tiles_single_tile_for_small_region(inspect_env):
wins, note = plan_tiles(
(0, 0, 1000, 500), 100.0, "text", (2000, 1414),
max_tiles=4, overlap=0.0, margin=0.02,
)
assert len(wins) == 1
assert "1×1" in note
# 目标框被外扩 5% 留边
assert wins[0]["min_x"] < 0 and wins[0]["max_x"] > 1000
def test_plan_tiles_splits_wide_region(inspect_env):
# 66000 世界单位宽(含 5% 留边)/ 单张 32000 → 3 张
wins, note = plan_tiles(
(0, 0, 60000, 20000), 100.0, "text", (2000, 1414),
max_tiles=9, overlap=0.0, margin=0.02,
)
assert len(wins) == 3
assert "3 张" in note
def test_plan_tiles_fatal_when_over_cap(inspect_env):
wins, note = plan_tiles(
(0, 0, 100000, 100000), 100.0, "text", (2000, 1414),
max_tiles=4, overlap=0.0, margin=0.02,
)
assert wins is None
assert "超过单次上限 4" in note
assert "span" in note # 给出缩小 target 的具体指引
def test_plan_tiles_falls_back_to_single_without_scale(inspect_env):
wins, note = plan_tiles(
(0, 0, 1000, 500), 0.0, "text", (2000, 1414),
max_tiles=4, overlap=0.0, margin=0.02,
)
assert len(wins) == 1
assert "退回单张" in note
def test_plan_tiles_degenerate_box_is_fatal(inspect_env):
wins, note = plan_tiles(
(0, 0, 0, 0), 100.0, "text", (2000, 1414),
max_tiles=4, overlap=0.0, margin=0.02,
)
assert wins is None
assert "退化" in note
# ---------------------------------------------------------------------------
# inspect_region(执行体)
# ---------------------------------------------------------------------------
def test_inspect_region_handle_success(inspect_env):
pytest.importorskip("PIL")
client = _client()
ctx = make_ctx(client=client)
ev = _evidence()
state = InspectState(ev)
content, budget = inspect_region(ctx, ev, {"target": {"handle": "A1"}}, [6], state)
assert budget[0] == 5 # 只用掉 1 张
assert "实体 A1" in content[0]["text"]
assert any(c.get("type") == "image_url" for c in content)
assert "zoom_to_window" in client.called_names()
assert "inspect-1" in state.labels
# 登记的可见范围应**包含**目标框(ZOOM W 等比适配会露出更多)
vis = state.labels["inspect-1"]
assert vis[0] <= 0.0 and vis[2] >= 1000.0
def test_inspect_region_label_target_reports_label_and_range(inspect_env):
pytest.importorskip("PIL")
ctx = make_ctx(client=_client())
ev = _evidence()
state = InspectState(ev)
content, budget = inspect_region(
ctx, ev, {"target": {"label": "overall", "cx": 0.5, "cy": 0.5, "span": 0.5}}, [6], state
)
assert budget[0] == 3 # 中心半幅需 3 张瓦片
assert "overall" in content[0]["text"]
assert any("可见世界范围" in c.get("text", "") for c in content)
def test_inspect_region_purpose_line_is_offered(inspect_env):
pytest.importorskip("PIL")
ctx = make_ctx(client=_client())
ev = _evidence()
content, _ = inspect_region(
ctx, ev, {"target": {"handle": "A1"}, "purpose": "line"}, [6], InspectState(ev)
)
assert "line" in content[0]["text"]
def test_inspect_region_bad_target_makes_no_mcp_calls(inspect_env):
client = _client()
ctx = make_ctx(client=client)
ev = _evidence()
content, budget = inspect_region(
ctx, ev, {"target": {"label": "nope"}}, [6], InspectState(ev)
)
assert budget[0] == 6
assert "未执行" in content[0]["text"]
assert client.called_names() == []
def test_inspect_region_oversized_target_makes_no_mcp_calls(inspect_env):
client = _client()
ctx = make_ctx(client=client)
ev = _evidence()
content, budget = inspect_region(
ctx, ev,
{"target": {"window": {"min_x": 0, "min_y": 0, "max_x": 200000, "max_y": 200000}}},
[6], InspectState(ev),
)
assert budget[0] == 6
assert "超过单次上限" in content[0]["text"]
assert client.called_names() == []
def test_inspect_region_budget_exhausted_returns_no_images(inspect_env):
pytest.importorskip("PIL")
ctx = make_ctx(client=_client())
ev = _evidence()
content, budget = inspect_region(
ctx, ev, {"target": {"handle": "A1"}}, [0], InspectState(ev)
)
assert budget[0] == 0
assert all(c.get("type") == "text" for c in content)
assert any("预算已用完" in c.get("text", "") for c in content)
# ---------------------------------------------------------------------------
# 与审查子循环的集成
# ---------------------------------------------------------------------------
def test_review_frame_can_call_inspect_region(inspect_env):
pytest.importorskip("PIL")
llm = FakeLlmClient(script=[
llm_calls([("inspect_region", {"target": {"handle": "A1"}})]),
llm_text('{"summary": "结论", "issues": []}'),
])
client = _client()
ctx = make_ctx(client=client, llm=llm)
ev = _evidence()
out = review_frame(ctx, ev, "sys", log=lambda *a: None)
assert out["summary"] == "结论"
names = client.called_names()
assert "zoom_to_window" in names # inspect_region 内部真的缩放了
assert names.count("zoom_to_window") == 2 # 放大 1 次 + 结束复位 1 次
def test_review_frame_offers_inspect_region_to_llm(inspect_env):
llm = FakeLlmClient(script=[llm_text("x")])
ctx = make_ctx(client=FakeMcpClient(), llm=llm)
review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
names = {t["function"]["name"] for t in llm.calls[0]["tools"]}
assert "inspect_region" in names
assert "zoom_to_window" not in names
+61
View File
@@ -2,9 +2,13 @@
import json
import pytest
from dwg_review import config
from dwg_review.cad_tools import READ_ONLY_TOOLS
from dwg_review.mcp_client import McpToolError
from dwg_review.phases.review import (
_review_tools,
_run_readonly_tool,
evidence_payload,
review_frame,
@@ -339,3 +343,60 @@ def test_review_frame_no_viewport_reset_without_window():
ev["window"] = None # 非 CAD 源(无窗口)→ 无从复位
review_frame(ctx, ev, "sys", log=lambda *a: None)
assert client.called_names().count("zoom_to_window") == 1
# ---------------------------------------------------------------------------
# 虚拟工具 inspect_region 的接入
# ---------------------------------------------------------------------------
def _tool_names(ctx):
return {t["function"]["name"] for t in _review_tools(ctx)}
def test_review_tools_swap_raw_view_tools_for_inspect_region(monkeypatch):
monkeypatch.setattr(config, "INSPECT_ENABLED", True)
names = _tool_names(make_ctx())
assert "inspect_region" in names
assert "query_entities" in names # 其余只读工具照旧
assert "zoom_to_window" not in names
assert "zoom_to_entity" not in names
assert "get_viewport_screenshot" not in names
def test_review_tools_keep_raw_view_tools_when_disabled(monkeypatch):
monkeypatch.setattr(config, "INSPECT_ENABLED", False)
names = _tool_names(make_ctx())
assert "inspect_region" not in names
assert {"zoom_to_window", "zoom_to_entity", "get_viewport_screenshot"} <= names
def test_readonly_intercepts_inspect_region_before_whitelist(monkeypatch):
# inspect_region 不在 READ_ONLY_TOOLS 里,但必须被**先**拦截(不能判成越权)
monkeypatch.setattr(config, "INSPECT_ENABLED", True)
ctx = make_ctx(client=FakeMcpClient())
ev = make_evidence()
from dwg_review.inspect import InspectState
content, budget = _run_readonly_tool(
ctx, "inspect_region", {"target": {}}, [6], evidence=ev, state=InspectState(ev)
)
assert "不在只读取证白名单内" not in content[0]["text"]
assert "未执行" in content[0]["text"] # 空 target → 明确的用法错误
assert budget == [6]
def test_readonly_inspect_region_unavailable_when_disabled(monkeypatch):
monkeypatch.setattr(config, "INSPECT_ENABLED", False)
ctx = make_ctx(client=FakeMcpClient())
ev = make_evidence()
from dwg_review.inspect import InspectState
content, _ = _run_readonly_tool(
ctx, "inspect_region", {}, [6], evidence=ev, state=InspectState(ev)
)
assert "不可用" in content[0]["text"]
def test_readonly_inspect_region_unavailable_without_evidence():
ctx = make_ctx(client=FakeMcpClient())
content, _ = _run_readonly_tool(ctx, "inspect_region", {}, [6])
assert "不可用" in content[0]["text"]
+59
View File
@@ -1,5 +1,7 @@
"""`dwg_review.tiling` 单元测试(纯函数,不依赖 CAD / LLM / Pillow)。"""
import pytest
from dwg_review import tiling as tl
@@ -221,3 +223,60 @@ def test_plan_regions_clamps_to_frame():
for w in wins:
assert w["min_x"] >= 0 and w["max_x"] <= 1000
assert w["min_y"] >= 0 and w["max_y"] <= 500
# --- box / window 互转与可见范围 -------------------------------------------
def test_box_window_roundtrip():
box = (1.0, 2.0, 3.0, 4.0)
win = tl.box_to_window(box)
assert win == {"min_x": 1.0, "min_y": 2.0, "max_x": 3.0, "max_y": 4.0}
assert tl.window_to_box(win) == box
def test_box_to_window_invalid():
assert tl.box_to_window(None) is None
assert tl.box_to_window([1, 2, 3]) is None
def test_window_to_box_invalid():
assert tl.window_to_box(None) is None
assert tl.window_to_box({}) is None
assert tl.window_to_box({"min_x": 1, "min_y": 2}) is None
def test_expand_box_by_fraction():
assert tl.expand_box((0, 0, 100, 50), 0.05) == pytest.approx((-5.0, -2.5, 105.0, 52.5))
assert tl.expand_box((0, 0, 100, 50), 0.0) == (0.0, 0.0, 100.0, 50.0)
assert tl.expand_box(None, 0.1) is None
# 非法 frac 视为 0(不抛异常)
assert tl.expand_box((0, 0, 10, 10), "x") == (0.0, 0.0, 10.0, 10.0)
def test_visible_box_widens_to_viewport_aspect():
# 1000x500 的窗口在 1280x280(约 4.57:1)视口下等比适配 → 横向露出更多,纵向刚好贴合
vis = tl.visible_box((0, 0, 1000, 500), (1280, 280), 0.0)
assert vis[0] < 0 and vis[2] > 1000
assert vis[1] == pytest.approx(0.0)
assert vis[3] == pytest.approx(500.0)
assert (vis[0] + vis[2]) / 2 == pytest.approx(500.0) # 中心不变
assert (vis[1] + vis[3]) / 2 == pytest.approx(250.0)
def test_visible_box_tall_window_is_fit_by_height():
# 100x1000(很窄很高)在 1280x280 视口下按**高**适配 → 纵向刚好贴合,横向大幅露出
vis = tl.visible_box((0, 0, 100, 1000), (1280, 280), 0.0)
assert vis[1] == pytest.approx(0.0) and vis[3] == pytest.approx(1000.0)
assert vis[0] < 0 and vis[2] > 100
assert (vis[0] + vis[2]) / 2 == pytest.approx(50.0) # 中心不变
def test_visible_box_without_view_px_falls_back_to_margin():
assert tl.visible_box((0, 0, 100, 50), None, 0.1) == pytest.approx(
(-10.0, -5.0, 110.0, 55.0)
)
assert tl.visible_box((0, 0, 100, 50), (0, 0), 0.0) == (0.0, 0.0, 100.0, 50.0)
def test_visible_box_invalid_box():
assert tl.visible_box(None, (100, 100), 0.1) is None