feat(review): 取证层多源化 + 视觉多图（整体+2x2 象限）
- 新增 evidence_model.py：统一证据模型（source_type + images 列表），访问器兼容旧 image_b64/image_mime 单图字段 - phases/evidence.py：新增 collect() 多源分发 seam（仅 CAD）、capture_images() 整体+等分象限多图（结束复位视口）、sub_windows() - phases/review.py：多图逐张下采样喂 LLM，payload 增加 source_type - config.py：EVIDENCE_DETAIL_TILES=4 / EVIDENCE_MAX_IMAGES=5（默认开启多图） - 新增 tests/test_evidence_model.py，更新 evidence/review 断言与 fakes 归一
This commit is contained in:
gjm committed
1 parent
22db93b6f5
commit
1c9fc72839
10 files changed
+396
-32
No files matched your search
@@ -1,14 +1,20 @@
|
||||
"""阶段 2：逐图框取证（确定性、串行）。
|
||||
"""阶段 2：逐审查单元取证（多源证据层 + 视觉多图）。
|
||||
|
||||
对每个图框：缩放到该框 → 取结构化数据（实体/文字/标注/块/多段线）→ 截取视口。
|
||||
对每个审查单元：取结构化数据（实体/文字/标注/块/多段线）+ 视觉截图（整体 + 可选局部）。
|
||||
结构化数据是权威依据，截图仅作视觉上下文。
|
||||
|
||||
多源设计：`collect()` 按审查单元来源分发；当前仅实现 CAD 源，PDF / 图片源预留 seam
|
||||
（未来在对应分支产出**同形** evidence，下游无需改动）。
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
|
||||
from .. import config
|
||||
from .. import config, evidence_model
|
||||
from ..mcp_client import result_images, result_json
|
||||
|
||||
_LABEL_OVERALL = "overall"
|
||||
|
||||
|
||||
def bbox_to_window(bbox):
|
||||
"""[minx,miny,maxx,maxy] -> window 对象。"""
|
||||
@@ -20,8 +26,77 @@ def bbox_to_window(bbox):
|
||||
}
|
||||
|
||||
|
||||
def sub_windows(window, tiles):
|
||||
"""把 window 等分为 tiles 个子窗口（tiles=4 → 2x2 象限）。
|
||||
|
||||
tiles<=1 返回 []。返回 [{"min_x",..,"max_x",..}, ...]，行优先（左上→右上→左下→右下）。
|
||||
"""
|
||||
if not tiles or tiles <= 1:
|
||||
return []
|
||||
cols = int(math.ceil(math.sqrt(tiles)))
|
||||
rows = int(math.ceil(tiles / cols))
|
||||
min_x, min_y = window["min_x"], window["min_y"]
|
||||
max_x, max_y = window["max_x"], window["max_y"]
|
||||
w = (max_x - min_x) / cols
|
||||
h = (max_y - min_y) / rows
|
||||
out = []
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
if len(out) >= tiles:
|
||||
break
|
||||
out.append(
|
||||
{
|
||||
"min_x": min_x + c * w,
|
||||
"min_y": max_y - (r + 1) * h,
|
||||
"max_x": min_x + (c + 1) * w,
|
||||
"max_y": max_y - r * h,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _zoom_and_shot(ctx, window, margin=0.02):
|
||||
"""缩放到窗口、等待渲染、截一张图，返回 (b64, mime) 或 None。"""
|
||||
ctx.tools.call("zoom_to_window", dict(window, margin=margin))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
shot = ctx.tools.call("get_viewport_screenshot", {})
|
||||
images = result_images(shot)
|
||||
if not images:
|
||||
return None
|
||||
return images[0]
|
||||
|
||||
|
||||
def capture_images(ctx, window, *, tiles=0, max_images=None):
|
||||
"""采集视觉证据：整体一张 + 局部 tiles 张。
|
||||
|
||||
- 整体 label="overall"，局部 label="tile-1"..；行优先。
|
||||
- 结束时**复位到整体视口**，保证后续流程从整幅视图起手。
|
||||
- 受 max_images（默认 config.EVIDENCE_MAX_IMAGES）截断。
|
||||
"""
|
||||
if max_images is None:
|
||||
max_images = config.EVIDENCE_MAX_IMAGES
|
||||
images = []
|
||||
|
||||
def _add(shot, label):
|
||||
if shot and len(images) < max_images:
|
||||
images.append(evidence_model.make_image(shot[0], shot[1], label))
|
||||
|
||||
# 整体
|
||||
_add(_zoom_and_shot(ctx, window), _LABEL_OVERALL)
|
||||
|
||||
# 局部（等分象限）
|
||||
for i, sub in enumerate(sub_windows(window, tiles), 1):
|
||||
if len(images) >= max_images:
|
||||
break
|
||||
_add(_zoom_and_shot(ctx, sub), f"tile-{i}")
|
||||
|
||||
# 复位整体视口（不截图）
|
||||
ctx.tools.call("zoom_to_window", dict(window, margin=0.02))
|
||||
return images
|
||||
|
||||
|
||||
def collect_frame(ctx, index, frame):
|
||||
"""采集单个图框的证据；bbox 无效时返回 None。"""
|
||||
"""采集单个图框（CAD 源）的证据；bbox 无效时返回 None。"""
|
||||
bbox = frame.get("bbox")
|
||||
if not bbox or len(bbox) != 4:
|
||||
return None
|
||||
@@ -29,11 +104,7 @@ def collect_frame(ctx, index, frame):
|
||||
win = bbox_to_window(bbox)
|
||||
limit = ctx.frame_limit
|
||||
|
||||
# a. 缩放到图框（给 CAD 留渲染时间）
|
||||
ctx.tools.call("zoom_to_window", dict(win, margin=0.02))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
|
||||
# b. 结构化数据（权威）
|
||||
# a. 结构化数据（权威；按 window 查询，不依赖缩放）
|
||||
entities = result_json(ctx.tools.call("query_entities", {"window": win, "limit": limit}), {})
|
||||
texts = result_json(ctx.tools.call("get_text_content", {"window": win, "limit": limit}), {})
|
||||
dimensions = result_json(ctx.tools.call("get_dimensions", {"window": win, "limit": limit}), {})
|
||||
@@ -44,12 +115,11 @@ def collect_frame(ctx, index, frame):
|
||||
ctx.tools.call("get_polylines_info", {"window": win, "limit": limit}), {}
|
||||
)
|
||||
|
||||
# c. 视口截图（视觉上下文）
|
||||
shot = ctx.tools.call("get_viewport_screenshot", {})
|
||||
images = result_images(shot)
|
||||
image_b64, image_mime = images[0] if images else (None, None)
|
||||
# b. 视觉：整体 + 可选局部（多图）
|
||||
images = capture_images(ctx, win, tiles=config.EVIDENCE_DETAIL_TILES)
|
||||
|
||||
return {
|
||||
"source_type": evidence_model.source_type_of(frame),
|
||||
"index": index,
|
||||
"frame": frame,
|
||||
"window": win,
|
||||
@@ -58,6 +128,18 @@ def collect_frame(ctx, index, frame):
|
||||
"dimensions": dimensions,
|
||||
"blocks": blocks,
|
||||
"polylines": polylines,
|
||||
"image_b64": image_b64,
|
||||
"image_mime": image_mime,
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
def collect(ctx, index, unit):
|
||||
"""按审查单元来源分发取证。
|
||||
|
||||
当前仅实现 CAD 源；PDF / 图片源预留（未来在对应分支接入离线解析 / 直接读图，
|
||||
产出与 CAD **同形**的 evidence：window=None、结构化键给空信封、images 带 label）。
|
||||
"""
|
||||
source = evidence_model.source_type_of(unit)
|
||||
if source == evidence_model.SOURCE_CAD:
|
||||
return collect_frame(ctx, index, unit)
|
||||
# TODO(pdf/image): 接入离线解析（DWG/PDF）与直接读图。
|
||||
return None
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
import json
|
||||
|
||||
from .. import evidence_model
|
||||
from ..cad_tools import READ_ONLY_TOOLS, is_write_tool
|
||||
from ..findings import parse_conclusion
|
||||
from ..llm_client import LlmClient, downsample_image_b64
|
||||
@@ -24,6 +25,7 @@ def evidence_payload(evidence):
|
||||
"""把证据整理成给 LLM 的 JSON（剔除 base64 图片，避免撑爆上下文）。"""
|
||||
frame = evidence.get("frame") or {}
|
||||
return {
|
||||
"source_type": evidence.get("source_type"),
|
||||
"frame_index": evidence.get("index"),
|
||||
"frame": {
|
||||
k: frame.get(k)
|
||||
@@ -72,9 +74,11 @@ def review_frame(ctx, evidence, system_prompt, log=print):
|
||||
+ json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
}
|
||||
]
|
||||
image_b64 = evidence.get("image_b64")
|
||||
if image_b64:
|
||||
nb, nm = downsample_image_b64(image_b64, evidence.get("image_mime") or "image/png")
|
||||
for img in evidence_model.evidence_images(evidence):
|
||||
nb, nm = downsample_image_b64(img["b64"], img.get("mime") or "image/png")
|
||||
label = img.get("label")
|
||||
if label:
|
||||
user_content.append({"type": "text", "text": f"（截图：{label}）"})
|
||||
user_content.append(LlmClient.image_content(nb, nm))
|
||||
|
||||
messages = [
|
||||
|
||||
Reference in new issue
Block a user