fix(mcp): 截图去除 CAD 界面元素（视口控件/ViewCube/滚动条）
- get_viewport_screenshot 改用 acedGetAcadDwgView() 只截图形区（排除滚动条/状态栏） - 截图期间临时置 NAVVCUBEDISPLAY=0、VPCONTROL=0 隐藏 ViewCube 与视口控件， RAII 守卫自动恢复原值，不影响用户 CAD 界面设置；配合 RedrawWindow 同步重绘 - 新增 verify_grounding.py：grounding 实测探针（截整体图 + VLM 定位放大区域）
This commit is contained in:
gjm committed
1 parent
32d552f1f0
commit
f8bd12dcec
2 files changed
+194
-6
No files matched your search
@@ -129,6 +129,42 @@ namespace
|
||||
|
||||
return ::GetForegroundWindow() == hwnd;
|
||||
}
|
||||
|
||||
// 临时把某个整型系统变量置 0，析构时恢复原值（RAII）。
|
||||
// 用于截图期间临时隐藏 ViewCube / 视口控件等叠加 UI，避免污染截图内容。
|
||||
// 若原值本就为 0（或变量不存在/取值失败），则不做任何改动。
|
||||
class TempSysVarOff
|
||||
{
|
||||
public:
|
||||
explicit TempSysVarOff(const ACHAR *name) : m_name(name), m_changed(false), m_old(0)
|
||||
{
|
||||
struct resbuf rb;
|
||||
if (acedGetVar(m_name, &rb) == RTNORM && rb.restype == RTSHORT && rb.resval.rint != 0)
|
||||
{
|
||||
m_old = rb.resval.rint;
|
||||
struct resbuf val;
|
||||
val.restype = RTSHORT;
|
||||
val.resval.rint = 0;
|
||||
m_changed = (acedSetVar(m_name, &val) == RTNORM);
|
||||
}
|
||||
}
|
||||
// 恢复原值；幂等（可显式提前调用，析构时兜底）
|
||||
void restore()
|
||||
{
|
||||
if (!m_changed) return;
|
||||
struct resbuf val;
|
||||
val.restype = RTSHORT;
|
||||
val.resval.rint = m_old;
|
||||
acedSetVar(m_name, &val);
|
||||
m_changed = false;
|
||||
}
|
||||
~TempSysVarOff() { restore(); }
|
||||
|
||||
private:
|
||||
const ACHAR *m_name;
|
||||
bool m_changed;
|
||||
int m_old;
|
||||
};
|
||||
}
|
||||
|
||||
class mcp_Tool_ViewportScreenshot: public mcp_Tool
|
||||
@@ -136,8 +172,14 @@ class mcp_Tool_ViewportScreenshot: public mcp_Tool
|
||||
public:
|
||||
virtual nlohmann::json Execute(const nlohmann::json &args) override
|
||||
{
|
||||
// 1. 获取 AutoCAD 当前文档的绘图区句柄
|
||||
HWND hWnd = adsw_acadDocWnd();
|
||||
// 1. 获取绘图视图窗口句柄（仅图形区，不含滚动条 / 状态栏等窗口边框）。
|
||||
// 优先 acedGetAcadDwgView()，取不到时退回文档窗口 adsw_acadDocWnd()。
|
||||
HWND hWnd = NULL;
|
||||
CView *pDwgView = acedGetAcadDwgView();
|
||||
if (pDwgView)
|
||||
hWnd = pDwgView->m_hWnd;
|
||||
if (!hWnd)
|
||||
hWnd = adsw_acadDocWnd();
|
||||
if (!hWnd)
|
||||
return mcp_Tool_Utility::make_error(-32000, L"无法获取CAD绘图区窗口句柄");
|
||||
|
||||
@@ -152,7 +194,16 @@ public:
|
||||
}
|
||||
bool stillOccluded = IsWindowOccluded(hWnd);
|
||||
|
||||
// 3. 初始化 GDI+
|
||||
// 3. 截图期间临时隐藏叠加 UI：ViewCube(NAVVCUBEDISPLAY) 与视口控件(VPCONTROL)。
|
||||
// 这两个系统变量控制绘图区右上角 / 左上角的浮层控件，会污染截图并误导视觉模型。
|
||||
// 守卫离开作用域时自动恢复原值，不影响用户的 CAD 界面设置。
|
||||
TempSysVarOff hideNavCube(L"NAVVCUBEDISPLAY");
|
||||
TempSysVarOff hideVpControl(L"VPCONTROL");
|
||||
// GDI 抓屏读的是屏幕像素：必须同步重绘，让叠加 UI 的隐藏立即反映到画面。
|
||||
::RedrawWindow(hWnd, NULL, NULL,
|
||||
RDW_INVALIDATE | RDW_UPDATENOW | RDW_ALLCHILDREN);
|
||||
|
||||
// 4. 初始化 GDI+
|
||||
Gdiplus::GdiplusStartupInput gdiplusStartupInput;
|
||||
ULONG_PTR gdiplusToken;
|
||||
Gdiplus::GdiplusStartup(&gdiplusToken, &gdiplusStartupInput, NULL);
|
||||
@@ -160,7 +211,7 @@ public:
|
||||
std::string base64Result;
|
||||
bool success = false;
|
||||
|
||||
// 4. 开始 Windows GDI 抓屏逻辑 (使用花括号限定生命周期)
|
||||
// 5. 开始 Windows GDI 抓屏逻辑 (使用花括号限定生命周期)
|
||||
{
|
||||
RECT rc;
|
||||
GetClientRect(hWnd, &rc);
|
||||
@@ -213,10 +264,16 @@ public:
|
||||
ReleaseDC(hWnd, hdcScreen);
|
||||
}
|
||||
|
||||
// 5. 关闭 GDI+
|
||||
// 6. 关闭 GDI+
|
||||
Gdiplus::GdiplusShutdown(gdiplusToken);
|
||||
|
||||
// 6. 按照 MCP 多模态协议返回数据
|
||||
// 7. 恢复叠加 UI（ViewCube / 视口控件）并重绘一次，让界面回到用户设置的样子
|
||||
hideVpControl.restore();
|
||||
hideNavCube.restore();
|
||||
::RedrawWindow(hWnd, NULL, NULL,
|
||||
RDW_INVALIDATE | RDW_UPDATENOW | RDW_ALLCHILDREN);
|
||||
|
||||
// 8. 按照 MCP 多模态协议返回数据
|
||||
if (success)
|
||||
{
|
||||
std::wstring msg = L"已成功截取 AutoCAD 当前视口画面。";
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""实测 deepseek-flash 的 grounding：能否给出可用的「放大区域」归一化坐标。
|
||||
|
||||
流程：截一张真实图框整体图 → 问 VLM「哪些区域需要放大」→ 解析它给的归一化坐标
|
||||
→ 映射成 CAD 世界坐标（WCS）→ 按坐标放大截图，供目视验证 grounding 是否可用。
|
||||
|
||||
用法：
|
||||
python verify_grounding.py [--frames 1] [--out reports/_probe]
|
||||
|
||||
前置：CAD 在线（127.0.0.1:8080 监听）、DEEPSEEK_API_KEY 可用。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from dwg_review import config
|
||||
from dwg_review.findings import extract_json_block
|
||||
from dwg_review.llm_client import LlmClient, downsample_image_b64
|
||||
from dwg_review.mcp_client import result_images
|
||||
from dwg_review.phases import planning
|
||||
from dwg_review.context import ReviewContext
|
||||
|
||||
_PROMPT = (
|
||||
"这是某工程图纸中一个图框的截图。请判断：图中哪些区域**细节不清、需要放大**才能看清？\n"
|
||||
"用**归一化坐标**给出想放大的区域：x0,y0,x1,y1 取值 0~1，"
|
||||
"图片左上角为 (0,0)、右下角为 (1,1)。\n"
|
||||
"最多给 3 个区域，按重要性排序。\n"
|
||||
'只输出 JSON，不要解释：{"regions":[{"bbox":[x0,y0,x1,y1],"reason":"..."}]}'
|
||||
)
|
||||
|
||||
|
||||
def _save(b64, path):
|
||||
path.write_bytes(base64.b64decode(b64))
|
||||
return path
|
||||
|
||||
|
||||
def _norm_to_wcs(win, x0, y0, x1, y1):
|
||||
"""归一化坐标（左上原点）→ WCS window。"""
|
||||
dx = win["max_x"] - win["min_x"]
|
||||
dy = win["max_y"] - win["min_y"]
|
||||
return {
|
||||
"min_x": win["min_x"] + x0 * dx,
|
||||
"max_x": win["min_x"] + x1 * dx,
|
||||
"min_y": win["max_y"] - y1 * dy, # 图片 y 向下，WCS y 向上
|
||||
"max_y": win["max_y"] - y0 * dy,
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--frames", type=int, default=1)
|
||||
ap.add_argument("--out", default="reports/_probe")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
ctx = ReviewContext.create()
|
||||
plan = planning.plan(ctx)
|
||||
frames = plan.get("frames", [])[: args.frames]
|
||||
if not frames:
|
||||
print("未识别到图框")
|
||||
return 1
|
||||
|
||||
out_dir = Path(config.PROJECT_ROOT) / args.out
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
frame = frames[0]
|
||||
win = {
|
||||
k: float(v)
|
||||
for k, v in zip(("min_x", "min_y", "max_x", "max_y"), frame["bbox"])
|
||||
}
|
||||
print("图框 window:", win)
|
||||
|
||||
ctx.tools.call("zoom_to_window", dict(win, margin=0.02))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
shot = ctx.tools.call("get_viewport_screenshot", {})
|
||||
imgs = result_images(shot)
|
||||
if not imgs:
|
||||
print("无截图")
|
||||
return 1
|
||||
b64, mime = imgs[0]
|
||||
nb, nm = downsample_image_b64(b64, mime)
|
||||
print("整体图:", _save(b64, out_dir / "overall.png"))
|
||||
|
||||
llm = LlmClient()
|
||||
print("模型:", llm.model)
|
||||
messages = [
|
||||
{"role": "system", "content": "你是资深工程图纸审图助手。"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": _PROMPT},
|
||||
LlmClient.image_content(nb, nm),
|
||||
],
|
||||
},
|
||||
]
|
||||
resp = llm.chat(messages)
|
||||
text = resp.choices[0].message.content or ""
|
||||
print("=== VLM 原始响应 ===")
|
||||
print(text)
|
||||
|
||||
block = extract_json_block(text)
|
||||
if not block:
|
||||
print("[X] 未解析出 JSON")
|
||||
return 1
|
||||
data = json.loads(block)
|
||||
regions = data.get("regions") or []
|
||||
print(f"=== 解析到 {len(regions)} 个区域 ===")
|
||||
|
||||
for i, r in enumerate(regions, 1):
|
||||
bbox = r.get("bbox")
|
||||
if not (isinstance(bbox, list) and len(bbox) == 4):
|
||||
print(f" 区域 {i}: bbox 非法 {bbox}")
|
||||
continue
|
||||
x0, y0, x1, y1 = (float(v) for v in bbox)
|
||||
sub = _norm_to_wcs(win, x0, y0, x1, y1)
|
||||
print(f" 区域 {i}: bbox={bbox} reason={r.get('reason')!r}")
|
||||
print(f" -> WCS {sub}")
|
||||
ctx.tools.call("zoom_to_window", dict(sub, margin=0.05))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
s2 = ctx.tools.call("get_viewport_screenshot", {})
|
||||
im2 = result_images(s2)
|
||||
if im2:
|
||||
print(" -> 已保存", _save(im2[0][0], out_dir / f"region_{i}.png"))
|
||||
|
||||
print("\n截图目录:", out_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user