feat(tiling): 自适应瓦片规划器 + VLM 分辨率离线标定

背景:逐图框审查固定「整体 + 2x2 象限」截图,大图幅(A0/A1)读不清细节。
改为按**实体分布**自适应决定截几张、截哪里。

- 新增 dwg_review/tiling.py(纯函数):
  SHEET_SIZES_MM / frame_world_size / frame_scale(sheet_scale>0 用之,否则由
  bbox/纸张反算,否则 default)/ effective_px(与 downsample 同规则)/
  world_per_px / is_text_type(标准名 + 自定义类名含 TEXT/LABEL/DIM/NOTE/TAG/
  ANNO/TITLE)/ cluster_text_flags / tiles_for_box / subdivide / plan_regions
  (文字簇优先、空白区不出图、max_tiles 截断、窗口夹到图框范围)
- 新增 verify_vlm_resolution_offline.py:PIL 合成类图纸图(精确像素字高 +
  可控线间距/线宽/虚线)→ 问 VLM 强约束 JSON → 定 R_text / R_line。
  实测:R_text=12px(黑底;白底 10px);R_line=6px。
  另:实测该模型**无法可靠计数**(6 条线在 2~10px 间距下答 4/10/4/5/4/4),
  故线条判据改用「1 条 vs 2 条」二选一。
- config 新增 TILING_ENABLED / TEXT_HEIGHT_MM / TEXT_READABLE_PX=12 /
  LINE_READABLE_PX=6 / TILE_MAX_COUNT / TILE_OVERLAP / HEATMAP_CELL_DIV /
  HEATMAP_CLUSTER_DILATE / HEATMAP_MAX_AREA_FRAC;EVIDENCE_DETAIL_TILES 与
  EVIDENCE_MAX_IMAGES 保留作回退路径参数
- llm_client 新增 image_size_b64(测量绘图区实际像素,决定单张覆盖范围)
- phases/evidence.capture_images 支持瓦片规划:先截整体图并测像素 →
  plan_regions 成功用规划窗口,否则退回 sub_windows(行为零变化);
  末尾始终复位整体视口。collect_frame 传实体/文字 bbox;任一信封
  total>returned(截断)→ 退回固定策略,避免误删有内容的瓦片
- 测试:tests/test_tiling.py(28 项)、tests/test_evidence.py 新增规划路径与
  降级用例、test_config / test_llm_client 补新常量与 image_size_b64;
  全量 283 项全绿
This commit is contained in:
gjm committed 2026-10-15 23:03:27 +08:00
1 parent 9b8a2329df
commit cba05b0c65
9 files changed
+1246 -10

No files matched your search

+26
View File
@@ -122,10 +122,36 @@ IMAGE_FORMAT = os.environ.get("IMAGE_FORMAT", "png").strip().lower()
IMAGE_JPEG_QUALITY = int(os.environ.get("IMAGE_JPEG_QUALITY", "80"))
# 取证阶段每图框的局部截图数(0=仅整体;4=整体+2x2 象限)
# —— 仅在自适应瓦片规划不可用时作为**回退**路径参数
EVIDENCE_DETAIL_TILES = int(os.environ.get("EVIDENCE_DETAIL_TILES", "4"))
# 每图框图片硬上限(整体 + 局部总张数)
EVIDENCE_MAX_IMAGES = int(os.environ.get("EVIDENCE_MAX_IMAGES", "5"))
# ---------------------------------------------------------------------------
# 自适应瓦片规划(按实体分布决定截几张、截哪里)
# ---------------------------------------------------------------------------
# 是否启用瓦片规划;False = 退回固定 EVIDENCE_DETAIL_TILES(行为与旧版一致)
TILING_ENABLED = _read_bool("TILING_ENABLED", True)
# 纸面字高(mm)。国标标注字高 2~3mm,取保守下限
TEXT_HEIGHT_MM = float(os.environ.get("TEXT_HEIGHT_MM", "2.0"))
# VLM 可读的**最小像素字高**。
# 离线标定(verify_vlm_resolution_offline.py,repeat=2):黑底 12px / 白底 10px 全行正确。
# CAD 模型空间默认黑底 → 取 12;离线合成图偏干净,故以 B2 CAD 实测为最终依据。
TEXT_READABLE_PX = int(os.environ.get("TEXT_READABLE_PX", "12"))
# VLM 可分辨的**最小线间距像素**。
# 同上标定:4~5px 有噪声、≥6px 稳定(1 条 vs 2 条二选一)。取 6 保守。
LINE_READABLE_PX = int(os.environ.get("LINE_READABLE_PX", "6"))
# 单图框瓦片数硬上界(成本兜底)
TILE_MAX_COUNT = int(os.environ.get("TILE_MAX_COUNT", "36"))
# 相邻瓦片重叠比例(防边界切字)
TILE_OVERLAP = float(os.environ.get("TILE_OVERLAP", "0.0"))
# 热力图网格划分(长边切几份)
HEATMAP_CELL_DIV = int(os.environ.get("HEATMAP_CELL_DIV", "32"))
# 热力图聚类前的膨胀格数(合并相近内容;0=不膨胀)
HEATMAP_CLUSTER_DILATE = int(os.environ.get("HEATMAP_CLUSTER_DILATE", "0"))
# 丢弃 bbox 面积超过图框该比例的实体(去图框边框/图签块)
HEATMAP_MAX_AREA_FRAC = float(os.environ.get("HEATMAP_MAX_AREA_FRAC", "0.5"))
def has_api_key() -> bool:
return bool(API_KEY.strip())
+15
View File
@@ -67,6 +67,21 @@ def downsample_image_b64(b64, mime="image/png", max_edge=None, fmt=None, quality
return b64, mime
def image_size_b64(b64, mime="image/png"):
"""解码图片并返回 `(宽, 高)`;缺 Pillow / 解码失败返回 None。
用于瓦片规划前测量**绘图区实际像素**(决定单张截图能覆盖多大世界范围)。
"""
if not HAS_PIL or not b64:
return None
try:
img = Image.open(io.BytesIO(base64.b64decode(b64)))
w, h = img.size
return (int(w), int(h))
except Exception:
return None
class LlmClient:
"""统一的 LLM 调用入口。"""
+97 -9
View File
@@ -10,7 +10,8 @@
import math
import time
from .. import config, evidence_model
from .. import config, evidence_model, heatmap, tiling
from ..llm_client import image_size_b64
from ..mcp_client import result_images, result_json
_LABEL_OVERALL = "overall"
@@ -66,10 +67,80 @@ def _zoom_and_shot(ctx, window, margin=0.02):
return images[0]
def capture_images(ctx, window, *, tiles=0, max_images=None):
"""采集视觉证据:整体一张 + 局部 tiles 张。
def _is_truncated(*envelopes):
"""任一信封 `total > len(items)` 即视为**被截断**(数据不全)。"""
for env in envelopes:
if not isinstance(env, dict):
continue
items = env.get("items")
if not isinstance(items, list):
continue
try:
total = int(env.get("total") or len(items))
except (TypeError, ValueError):
total = len(items)
if total > len(items):
return True
return False
def _text_boxes(entities, texts, dimensions):
"""收集「文字类」实体 bbox。
标准文字 / 标注信封(`get_text_content` / `get_dimensions`)全收;
再加**类名含 text/label/dim 等关键词**的自定义实体(`tiling.is_text_type`)。
"""
boxes = list(heatmap.entity_boxes(texts))
boxes.extend(heatmap.entity_boxes(dimensions))
items = entities.get("items") if isinstance(entities, dict) else None
for it in items or []:
if isinstance(it, dict) and tiling.is_text_type(it.get("type")):
b = heatmap.norm_box(it.get("bbox"))
if b is not None:
boxes.append(b)
return boxes
def _plan_tiles(window, frame, boxes, text_boxes, truncated, view_px):
"""尝试按实体分布规划瓦片窗口;任何条件不满足都返回 [](调用方退回固定策略)。
降级条件:规划被关闭 / 实体数据被截断 / 缺图框 / 无实体 / 拿不到绘图区像素 /
比例未知(`frame_scale` 返 0)。
"""
if not config.TILING_ENABLED or truncated or not frame or not boxes or not view_px:
return []
scale = tiling.frame_scale(frame)
s_text = tiling.world_per_px(scale, config.TEXT_HEIGHT_MM, config.TEXT_READABLE_PX)
# 线条区只需看清走向:同以「最小关注特征尺寸」(纸面 TEXT_HEIGHT_MM)为基准,
# 但像素门槛更低(R_line < R_text)→ 覆盖范围更大、瓦片更少。
s_line = tiling.world_per_px(scale, config.TEXT_HEIGHT_MM, config.LINE_READABLE_PX)
if s_text <= 0 or s_line <= 0:
return []
eff = tiling.effective_px(view_px, config.IMAGE_MAX_EDGE)
if eff[0] <= 0 or eff[1] <= 0:
return []
cell = heatmap.div_to_cell(window, config.HEATMAP_CELL_DIV)
grid = heatmap.build_grid(window, boxes, cell)
clusters = heatmap.cluster_cells(grid, dilate=config.HEATMAP_CLUSTER_DILATE)
if not clusters:
return []
flags = tiling.cluster_text_flags(clusters, text_boxes)
text_idx = {i for i, is_text in enumerate(flags) if is_text}
return tiling.plan_regions(
frame, clusters,
text_clusters=text_idx, s_text=s_text, s_line=s_line, view_px=eff,
max_tiles=config.TILE_MAX_COUNT, overlap=config.TILE_OVERLAP,
)
def capture_images(ctx, window, *, tiles=0, max_images=None, frame=None,
boxes=None, text_boxes=None, truncated=False):
"""采集视觉证据:整体一张 + 局部若干张(自适应瓦片优先,否则等分象限回退)。
- 整体 label="overall",局部 label="tile-1"..;行优先。
- 局部窗口优先由 `_plan_tiles` 按**实体分布**给出(空白区不出图);
不可用时退回 `sub_windows(window, tiles)`(行为与旧版完全一致)。
- 结束时**复位到整体视口**,保证后续流程从整幅视图起手。
- 受 max_images(默认 config.EVIDENCE_MAX_IMAGES)截断。
"""
@@ -81,11 +152,17 @@ def capture_images(ctx, window, *, tiles=0, max_images=None):
if shot and len(images) < max_images:
images.append(evidence_model.make_image(shot[0], shot[1], label))
# 整体
_add(_zoom_and_shot(ctx, window), _LABEL_OVERALL)
# 整体图:既是证据,也用于测量绘图区**实际像素**(决定单张截图的世界覆盖范围)
overall = _zoom_and_shot(ctx, window)
view_px = image_size_b64(overall[0]) if overall else None
_add(overall, _LABEL_OVERALL)
# 局部(等分象限)
for i, sub in enumerate(sub_windows(window, tiles), 1):
# 局部:优先自适应瓦片,失败退回等分象限
regions = _plan_tiles(window, frame, boxes, text_boxes, truncated, view_px)
if not regions:
regions = sub_windows(window, tiles)
for i, sub in enumerate(regions, 1):
if len(images) >= max_images:
break
_add(_zoom_and_shot(ctx, sub), f"tile-{i}")
@@ -115,8 +192,19 @@ def collect_frame(ctx, index, frame):
ctx.tools.call("get_polylines_info", {"window": win, "limit": limit}), {}
)
# b. 视觉:整体 + 可选局部(多图)
images = capture_images(ctx, win, tiles=config.EVIDENCE_DETAIL_TILES)
# b. 视觉:整体 + 局部(优先按实体分布自适应瓦片;数据截断则退回固定策略)
truncated = _is_truncated(entities, texts, dimensions, blocks, polylines)
boxes = heatmap.drop_oversized(
heatmap.entity_boxes(entities), win, config.HEATMAP_MAX_AREA_FRAC
)
images = capture_images(
ctx, win,
tiles=config.EVIDENCE_DETAIL_TILES,
frame=frame,
boxes=boxes,
text_boxes=_text_boxes(entities, texts, dimensions),
truncated=truncated,
)
return {
"source_type": evidence_model.source_type_of(frame),
+284
View File
@@ -0,0 +1,284 @@
"""自适应瓦片规划器:按实体分布把图框切成若干 zoom 窗口,使 VLM 能看清细节。
纯函数模块(仅依赖 `math`),不依赖 Pillow / CAD。
核心思路(用户方案):
1. 由图框纸张 + 比例换算「CAD 图形单位 / 像素」s;
2. 由 VLM 可读下限(文字 `R_text` px、线条 `R_line` px)定每张截图的**世界覆盖范围**;
3. 复用热力图(`heatmap.py`)的**占用网格 + 聚类**得到簇包围盒;簇内是否含**文字类实体**
决定用细(`s_text`)/ 粗(`s_line`)尺度;
4. 簇 ≤ 单屏 → 1 张;超出 → 行优先细分;**空白区不出图**(整体图已覆盖);
受 `max_tiles` 截断,**文字簇优先保留**。
坐标约定:与 `heatmap.py` / CAD 一致——x 向右、y **向上**;bbox=(min_x,min_y,max_x,max_y);
瓦片行优先且 row 0 在**最上方**(与 `phases/evidence.py::sub_windows` 一致)。
"""
import math
# 纸张尺寸(mm,长边在前):GB/T 14689 基本幅面
SHEET_SIZES_MM = {
"A0": (1189.0, 841.0),
"A1": (841.0, 594.0),
"A2": (594.0, 420.0),
"A3": (420.0, 297.0),
"A4": (297.0, 210.0),
}
# 标准实体名:本身就是文字/标注类
_STD_TEXT_TYPES = frozenset({"TEXT", "MTEXT", "ATTRIB", "ATTDEF", "DIMENSION"})
# 自定义实体类名关键词(用户经验:类名常含 text/label/dim 等)
_TEXT_KEYWORDS = ("TEXT", "LABEL", "DIM", "NOTE", "TAG", "ANNO", "TITLE")
# 瓦片重叠上限(防边界切字;过大会显著增加张数)
_MAX_OVERLAP = 0.5
def norm_box(b):
"""归一 bbox 为 (minx,miny,maxx,maxy);非法返回 None。"""
if not isinstance(b, (list, tuple)) or len(b) != 4:
return None
try:
x1, y1, x2, y2 = (float(v) for v in b)
except (TypeError, ValueError):
return None
if not all(math.isfinite(v) for v in (x1, y1, x2, y2)):
return None
if x1 > x2:
x1, x2 = x2, x1
if y1 > y2:
y1, y2 = y2, y1
return (x1, y1, x2, y2)
def frame_bbox(frame):
"""图框 bbox 元组;非法返回 None。"""
if not isinstance(frame, dict):
return None
box = norm_box(frame.get("bbox"))
if box is None or box[2] <= box[0] or box[3] <= box[1]:
return None
return box
def frame_world_size(frame):
"""图框在 CAD 中的世界尺寸 (W, H);bbox 非法返回 (0.0, 0.0)。"""
box = frame_bbox(frame)
if box is None:
return (0.0, 0.0)
return (box[2] - box[0], box[3] - box[1])
def frame_scale(frame, default=0.0):
"""图框出图比例(1:100 → 100.0)。
优先级:
1. `sheet_scale`(工具已识别,>0 直接用);
2. 由「图框世界尺寸 / 纸张尺寸」反算(需 `sheet_size` 命中纸张表);
3. `default`(拿不到 → 调用方退回固定策略)。
"""
if not isinstance(frame, dict):
return default
try:
s = float(frame.get("sheet_scale") or 0.0)
except (TypeError, ValueError):
s = 0.0
if s > 0:
return s
w, h = frame_world_size(frame)
if w <= 0 or h <= 0:
return default
sheet = SHEET_SIZES_MM.get(str(frame.get("sheet_size") or "").strip().upper())
if not sheet:
return default
sw, sh = sheet
# 判断横放 / 纵放:取长宽比更接近的朝向
ar = w / h
if abs(ar - sw / sh) <= abs(ar - sh / sw):
return w / sw # 横放:世界宽 ↔ 纸宽
return w / sh # 纵放:世界宽 ↔ 纸高
def effective_px(view_px, max_edge):
"""按长边下采样后的**有效像素** (ew, eh)。
与 `llm_client.downsample_image_b64` 的缩放规则一致(长边缩到 max_edge)。
非法输入返回 (0, 0);max_edge<=0 表示不缩放。
"""
if not view_px or len(view_px) != 2:
return (0, 0)
try:
w, h = int(view_px[0]), int(view_px[1])
except (TypeError, ValueError):
return (0, 0)
if w <= 0 or h <= 0:
return (0, 0)
if max_edge and max_edge > 0 and max(w, h) > max_edge:
k = max_edge / float(max(w, h))
return (max(1, int(round(w * k))), max(1, int(round(h * k))))
return (w, h)
def world_per_px(scale, text_mm, readable_px):
"""每像素代表的 CAD 图形单位:s = (text_mm * scale) / readable_px。
`text_mm` 为纸面字高(国标 2~3mm),`scale` 为出图比例(1:100 → 100),
`readable_px` 为 VLM 可读的**最小像素字高**。任一非正 → 返回 0.0(表示无法规划)。
"""
try:
scale, text_mm, readable_px = float(scale), float(text_mm), float(readable_px)
except (TypeError, ValueError):
return 0.0
if scale <= 0 or text_mm <= 0 or readable_px <= 0:
return 0.0
return text_mm * scale / readable_px
def is_text_type(type_name):
"""实体类型名是否为「文字/标注类」。
标准实体给 DXF 名(TEXT/MTEXT/ATTRIB/ATTDEF/DIMENSION)→ 精确命中;
自定义实体兜底为**类名去 AcDb 前缀并大写**,故按**关键词包含**判定
(TEXT/LABEL/DIM/NOTE/TAG/ANNO/TITLE)。非字符串/空 → False。
"""
if not isinstance(type_name, str):
return False
t = type_name.strip().upper()
if not t:
return False
if t in _STD_TEXT_TYPES:
return True
return any(k in t for k in _TEXT_KEYWORDS)
def boxes_intersect(a, b, eps=0.0):
"""两个 bbox 是否相交(接触视为相交;eps 可放宽)。"""
if a is None or b is None:
return False
return not (
a[2] < b[0] - eps or b[2] < a[0] - eps
or a[3] < b[1] - eps or b[3] < a[1] - eps
)
def cluster_text_flags(clusters, text_boxes):
"""标记每个簇是否含文字类实体(bbox 相交即算,保守)。
返回与 `clusters` 等长的 bool 列表。
"""
boxes = [b for b in (norm_box(x) for x in (text_boxes or [])) if b]
flags = []
for cl in clusters or []:
b = norm_box(cl.get("bbox")) if isinstance(cl, dict) else None
flags.append(bool(b) and any(boxes_intersect(b, tb) for tb in boxes))
return flags
def tiles_for_box(box, cap_w, cap_h, overlap=0.0):
"""覆盖 box 所需的 (cols, rows)。
`cap_w`/`cap_h` 为单张截图的世界覆盖范围;`overlap` 为相邻瓦片重叠比例
(>0 时有效步长缩短,张数增加)。非法输入返回 (1, 1)。
"""
if box is None:
return (1, 1)
w, h = box[2] - box[0], box[3] - box[1]
if w <= 0 or h <= 0:
return (1, 1)
ov = min(max(float(overlap or 0.0), 0.0), _MAX_OVERLAP)
step_w = float(cap_w) * (1.0 - ov)
step_h = float(cap_h) * (1.0 - ov)
if step_w <= 0 or step_h <= 0:
return (1, 1)
return (max(1, int(math.ceil(w / step_w))), max(1, int(math.ceil(h / step_h))))
def _clamp_window(win, frame_win):
"""把窗口夹到图框范围内;夹后退化(宽/高<=0)返回 None。"""
if frame_win is None:
return win
fx1, fy1, fx2, fy2 = frame_win
x1 = max(win["min_x"], fx1)
y1 = max(win["min_y"], fy1)
x2 = min(win["max_x"], fx2)
y2 = min(win["max_y"], fy2)
if x2 <= x1 or y2 <= y1:
return None
return {"min_x": x1, "min_y": y1, "max_x": x2, "max_y": y2}
def subdivide(box, cols, rows, overlap=0.0, frame_win=None):
"""把 box 等分为 cols×rows 个 zoom 窗口(行优先,row 0 在最上方)。
每张按 `overlap` 比例向外扩张(防边界切字),再夹到 `frame_win`。
"""
x1, y1, x2, y2 = box
cols, rows = max(1, int(cols)), max(1, int(rows))
w = (x2 - x1) / cols
h = (y2 - y1) / rows
ov = min(max(float(overlap or 0.0), 0.0), _MAX_OVERLAP)
mx, my = w * ov, h * ov
out = []
for r in range(rows):
for c in range(cols):
sx1 = x1 + c * w
sy2 = y2 - r * h # row 0 在最上方
win = {
"min_x": sx1 - mx, "min_y": sy2 - h - my,
"max_x": sx1 + w + mx, "max_y": sy2 + my,
}
clamped = _clamp_window(win, frame_win)
if clamped:
out.append(clamped)
return out
def plan_regions(frame, clusters, *, text_clusters=(), s_text=0.0, s_line=0.0,
view_px=(0, 0), max_tiles=36, overlap=0.0):
"""把图框内的簇规划成一组 zoom 窗口(自适应瓦片)。
参数:
- `frame` :图框(含 bbox / sheet_size / sheet_scale);
- `clusters` :`heatmap.cluster_cells` 的输出(已按实体计数降序);
- `text_clusters`:含文字类实体的**簇下标**集合(决定用 s_text 还是 s_line);
- `s_text`/`s_line`:CAD 图形单位 / 像素(`world_per_px` 的输出);
- `view_px` :单张截图的**有效像素** (ew, eh)(`effective_px` 的输出);
- `max_tiles` :返回窗口数硬上界(文字簇优先保留);
- `overlap` :相邻瓦片重叠比例。
返回行优先的 window 列表;**无法规划时返回 []**(调用方退回固定策略)。
"""
frame_win = frame_bbox(frame)
if frame_win is None:
return []
ew, eh = view_px[0] if view_px else 0, view_px[1] if view_px and len(view_px) > 1 else 0
if ew <= 0 or eh <= 0:
return []
text_idx = set(text_clusters or ())
# 文字簇优先(同级内保持输入的计数降序)
ordered = sorted(
enumerate(clusters or []),
key=lambda p: (0 if p[0] in text_idx else 1, p[0]),
)
regions = []
for i, cl in ordered:
if len(regions) >= max_tiles:
break
box = norm_box(cl.get("bbox")) if isinstance(cl, dict) else None
if box is None:
continue
s = s_text if i in text_idx else s_line
if s <= 0:
continue
cap_w, cap_h = s * ew, s * eh
cols, rows = tiles_for_box(box, cap_w, cap_h, overlap)
# 单屏可覆盖 → 1 张(subdivide 仍会按 overlap 外扩并夹到图框)
for win in subdivide(box, cols, rows, overlap, frame_win):
regions.append(win)
if len(regions) >= max_tiles:
break
return regions[:max_tiles]