diff --git a/.gitignore b/.gitignore index 2a597f9..9a25abf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ lib/ *.tmp *.aps *.diagsession +!tools/perf_diagsession.py +__pycache__/ diff --git a/tools/perf_diagsession.py b/tools/perf_diagsession.py new file mode 100644 index 0000000..a9bb81e --- /dev/null +++ b/tools/perf_diagsession.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +perf_diagsession.py — 把 .diagsession / .etl 的 CPU 采样解析成「哪个函数/模块吃 CPU」的报告。 + +场景:AutoCAD 打开大图纸慢,需要定位热点。采集方式二选一: + + # A) WPR(推荐,轻量、不引入 VS 抢核) + wpr -start CPU -filemode + ... 打开图纸,等界面完全空闲 + wpr -stop C:\\temp\\open.etl + + # B) Visual Studio:调试 -> 性能探查器 -> CPU 使用率 -> 附加到 acad.exe -> 打开图纸 + # 得到 .diagsession(本质是 zip,内含 sc.user_aux.etl) + +重要前提:**采完之后到分析完之前不要重新编译**。 +脚本会用 trace 里记录的模块大小 + PE TimeDateStamp 去 /bin 下找同一份二进制, +再找它旁边的 .pdb。找不到就说明符号对不上,我们的函数只能显示成「模块 + RVA」。 + +用法: + python tools/perf_diagsession.py + python tools/perf_diagsession.py open.etl --verify-only # 只校验符号,10 秒 + python tools/perf_diagsession.py open.etl --out report.md + python tools/perf_diagsession.py open.etl --net-symbols # 额外从微软符号服务器取系统符号 + +依赖: Windows Performance Toolkit 里的 xperf.exe;Python 3.8+(仅标准库)。 +""" + +import argparse +import collections +import glob +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import zipfile + +WPT_DIRS = [ + r"C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit", + r"C:\Program Files\Windows Kits\10\Windows Performance Toolkit", +] +SYMBOL_CACHE = r"C:\Symbols" +SYMCACHE_DIR = r"C:\SymCache" +MSDL = "srv*C:\\Symbols*https://msdl.microsoft.com/download/symbols" + +# 观察这些模块的「直接调用方」:match = 触发本次统计的模块前缀,skip = 找调用方时要跳过的前缀 +SITE_OF_INTEREST = { + "sqliteserver.dll": (("sqliteserver.dll",), ("sqliteserver.dll", "sqlite3_x64.dll")), + "sqlite3_x64.dll": (("sqlite3_x64.dll",), ("sqliteserver.dll", "sqlite3_x64.dll")), + "Parasolid": (("ASMKERN", "ASMBASE"), ("ASM", "tbb.dll")), +} +OUR_MODULE_HINT = ("CoreCommand", "AecRxBase", "CoreLibrary", "Dimension", "wfK", + "wfA", "wfB", "wfKernal", "ElecCmd", "HvacCmd", "PipelineCmd", + "ArchCmd", "EWMain", "EarthWorm", "ISODraft", "Plumbing", + "SPLD", "PIDLine", "PipeHanger", "ThunderProof", "AdsXRx", + "MFCCtrl", "sqliteserver") + + +def log(msg): + # 控制台编码在不同机器上可能是 cp936/utf-8,日志统一走 ASCII 避免乱码; + # 中文只出现在生成的 markdown 报告里(显式写 UTF-8)。 + try: + msg.encode("ascii") + except UnicodeEncodeError: + msg = msg.encode("ascii", "backslashreplace").decode("ascii") + print(msg, file=sys.stderr, flush=True) + + +# --------------------------------------------------------------------------- # +# 环境 / 输入 +# --------------------------------------------------------------------------- # +def find_xperf(): + for d in WPT_DIRS: + p = os.path.join(d, "xperf.exe") + if os.path.exists(p): + return p + for p in glob.glob(r"C:\Program Files*\Windows Kits\*\Windows Performance Toolkit\xperf.exe"): + return p + raise SystemExit("找不到 xperf.exe:请安装 Windows Performance Toolkit" + "(Windows SDK 的一个组件)") + + +def load_etl(path, workdir): + """返回 etl 路径;.diagsession 会解出里面最大的 .etl""" + if path.lower().endswith(".etl"): + log(f"[1/5] use etl directly: {path} ({os.path.getsize(path):,} bytes)") + return path + if not zipfile.is_zipfile(path): + raise SystemExit(f"{path} 既不是 .etl 也不是 zip 格式的 .diagsession") + z = zipfile.ZipFile(path) + cands = [n for n in z.namelist() if n.lower().endswith(".etl")] + if not cands: + raise SystemExit("diagsession 里没有 .etl(可能只录了内存/计数器数据源)") + src = max(cands, key=lambda n: z.getinfo(n).file_size) + dst = os.path.join(workdir, "trace.etl") + with z.open(src) as fi, open(dst, "wb") as fo: + shutil.copyfileobj(fi, fo) + log(f"[1/5] extract {src} -> {dst} ({os.path.getsize(dst):,} bytes)") + return dst + + +def build_env(repo, net_symbols): + env = dict(os.environ) + parts = [] + if repo: + for d in glob.glob(os.path.join(repo, "bin", "Code2026", "**"), recursive=True): + if os.path.isdir(d) and glob.glob(os.path.join(d, "*.pdb")): + parts.append(d) + if os.path.isdir(SYMBOL_CACHE): + parts.append(SYMBOL_CACHE) + if net_symbols: + parts.append(MSDL) + env["_NT_SYMBOL_PATH"] = ";".join(parts) + env["_NT_SYMCACHE_PATH"] = SYMCACHE_DIR + return env + + +# --------------------------------------------------------------------------- # +# 符号校验:trace 里的二进制 == bin 下的二进制 ? 旁边有 .pdb 吗 ? +# --------------------------------------------------------------------------- # +def pe_stamp(path): + """(TimeDateStamp, SizeOfImage) + + 注意:要和 trace 里的 EndAddr-BaseAddr 比就必须用 SizeOfImage(虚拟大小,0x1000 对齐), + 不能用文件大小——两者差几十 KB,之前就是这里判错导致所有模块都"不匹配"。 + """ + try: + with open(path, "rb") as f: + head = f.read(4096) + if head[:2] != b"MZ": + return None + e = struct.unpack_from(" (base, size, timestamp)""" + out = os.path.join(workdir, "images.txt") + with open(out, "w", encoding="utf-8", errors="replace") as fo: + subprocess.run([find_xperf(), "-i", etl, "-a", "process", "-image"], + stdout=fo, stderr=subprocess.DEVNULL, env=env) + res = {} + pat = re.compile(r'Image,.*?, 0x([0-9a-f]+), 0x([0-9a-f]+), 0x[0-9a-f]+,\s+0x([0-9a-f]+),.*?"([^"]+)",') + for line in open(out, encoding="utf-8", errors="replace"): + m = pat.search(line) + if m: + base, end = int(m.group(1), 16), int(m.group(2), 16) + res[m.group(4).lower()] = (base, end - base, int(m.group(3), 16)) + return res + + +def index_binaries(repo): + """name -> [ (path, ts, size) ]""" + idx = collections.defaultdict(list) + if not repo: + return idx + for f in glob.glob(os.path.join(repo, "bin", "Code2026", "**", "*"), recursive=True): + if not os.path.isfile(f) or not f.lower().endswith((".arx", ".dll", ".dbx", ".exe")): + continue + st = pe_stamp(f) + if st: + idx[os.path.basename(f).lower()].append((f, st[0], st[1])) + return idx + + +def verify_symbols(images, repo): + """返回 (matched, mismatched, no_pdb) + + 身份判定:先比 TimeDateStamp + SizeOfImage;若 trace 里 TimeDateStamp 为 0 + (部分目标不写时间戳),退化为只比 SizeOfImage。 + """ + idx = index_binaries(repo) + matched, mismatched, no_pdb = {}, [], [] + for name, (_, size, ts) in images.items(): + cands = idx.get(name) + if not cands: + continue + hit = None + for p, t, s in cands: + if s != size: + continue + if ts and t and t != ts: + continue + hit = p + break + if not hit: + mismatched.append(name) + continue + pdb = os.path.splitext(hit)[0] + ".pdb" + if os.path.exists(pdb): + matched[name] = (hit, pdb) + else: + no_pdb.append(name) + return matched, mismatched, no_pdb + + +# --------------------------------------------------------------------------- # +# 一趟 dumper,全部聚合 +# --------------------------------------------------------------------------- # +def aggregate(etl, env, target, top_sites=15): + cmd = [find_xperf(), "-i", etl, "-symbols", "-a", "dumper"] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=env, text=True, encoding="utf-8", errors="replace") + + excl_mod = collections.Counter() + excl_fn = collections.Counter() # "MOD!FUNC" + our_excl = collections.Counter() # 我们模块的叶子 + our_incl = collections.Counter() # 我们模块函数出现在栈中(每栈去重) + asm_drivers = collections.Counter() # 栈里有 Parasolid 时,第一个我们的帧 + incl_mod = collections.Counter() + threads = collections.Counter() + thread_leaf = collections.defaultdict(collections.Counter) + chains = collections.Counter() + sites = collections.defaultdict(collections.Counter) + allproc = collections.Counter() + sec_leaf = collections.defaultdict(collections.Counter) + + cur, tid, ts_us, frames = None, None, 0, [] + n_total = 0 + + def flush(): + nonlocal frames, n_total + if not frames or cur != target: + frames = [] + return + n_total += 1 + leaf_mod, leaf_fn, leaf_addr = frames[0] + excl_mod[leaf_mod] += 1 + excl_fn[f"{leaf_mod}!{leaf_fn}"] += 1 + threads[tid] += 1 + thread_leaf[tid][leaf_mod] += 1 + sec_leaf[ts_us // 1000000][leaf_mod] += 1 + if any(h.lower() in leaf_mod.lower() for h in OUR_MODULE_HINT): + our_excl[f"{leaf_mod}!{leaf_fn}"] += 1 + seen = set() + for m, f, _a in frames: + if m not in seen: + seen.add(m) + incl_mod[m] += 1 + seen_our = set() + for m, f, _a in frames: + if any(h.lower() in m.lower() for h in OUR_MODULE_HINT): + k = f"{m}!{f}" + if k not in seen_our: + seen_our.add(k) + our_incl[k] += 1 + if any(m.startswith(("ASMKERN", "ASMBASE")) for m in (fr[0] for fr in frames)): + for m, f, _a in frames: + if any(h.lower() in m.lower() for h in OUR_MODULE_HINT): + asm_drivers[f"{m}!{f}"] += 1 + break + coll = [] + for m, _f, _a in frames: + if not coll or coll[-1] != m: + coll.append(m) + chains[" <- ".join(coll[:11])] += 1 + + for label, (match_pfx, skip_pfx) in SITE_OF_INTEREST.items(): + start = next((i for i, fr in enumerate(frames) + if fr[0].startswith(match_pfx)), None) + if start is None: + continue + for j in range(start + 1, len(frames)): + m = frames[j][0] + if not any(m.startswith(p) for p in skip_pfx): + sites[label][f"{m}!{frames[j][1]}"] += 1 + break + frames = [] + + for line in proc.stdout: + if not line.strip(): + continue + ls = line.lstrip() + if ls.startswith("SampledProfile,"): + flush() + f = line.split(",", 4) + if len(f) < 4: + cur = None + continue + cur = re.sub(r"\s*\(\d+\)$", "", f[2].strip()) + allproc[cur] += 1 + try: + ts_us = int(f[1]) + except ValueError: + ts_us = 0 + tid = f[3].strip() + elif ls.startswith("Stack,"): + f = line.split(",", 5) + if len(f) >= 6 and cur == target: + mod, _, fn = f[5].strip().partition("!") + frames.append((mod.strip(), fn.strip(), f[4].strip())) + flush() + proc.wait() + + return dict( + samples=n_total, + excl_mod=excl_mod.most_common(40), + excl_fn=excl_fn.most_common(60), + our_excl=our_excl.most_common(30), + our_incl=our_incl.most_common(40), + asm_drivers=asm_drivers.most_common(25), + incl_mod=incl_mod.most_common(40), + threads=threads.most_common(15), + thread_leaf={t: thread_leaf[t].most_common(6) for t, _ in threads.most_common(12)}, + chains=chains.most_common(30), + sites={k: v.most_common(top_sites) for k, v in sites.items()}, + all_proc=allproc.most_common(10), + timeline=[(s, sum(sec_leaf[s].values()), sec_leaf[s].most_common(3)) + for s in sorted(sec_leaf)], + ) + + +# --------------------------------------------------------------------------- # +# 报告 +# --------------------------------------------------------------------------- # +def is_addr(fn): + return bool(re.fullmatch(r"0x[0-9a-fA-F]+", fn or "")) + + +def _strip_templates(s): + """删掉所有(可嵌套的)<...>,能正确处理 >>""" + out = [] + depth = 0 + for ch in s: + if ch == "<": + depth += 1 + elif ch == ">": + if depth: + depth -= 1 + elif depth == 0: + out.append(ch) + return "".join(out) + + +def short(name, limit=110): + """折叠 C++ 模板参数,否则 STL 容器名一行就几百字符看不清""" + s = re.sub(r"\s+", " ", name or "") + s = _strip_templates(s) + if len(s) > limit: + s = s[:limit] + "..." + return s + + +def addr_hint(mod, fn, images): + """无符号时把地址换算成 模块+RVA;基址未知时保留原地址,不算负 RVA""" + if not is_addr(fn): + return short(f"{mod}!{fn}") + base = images.get(mod.lower(), (None,))[0] + if base is None: + return f"{mod}!{fn} (无符号)" + rva = int(fn, 16) - base + if rva < 0: + return f"{mod}!{fn} (无符号)" + return f"{mod}!RVA 0x{rva:x} (无符号)" + + +def render(agg, images, sym, src_name, target, out_path): + n = agg["samples"] or 1 + matched, mismatched, no_pdb = sym + L = [] + a = L.append + + a(f"# CPU 热点报告 — `{os.path.basename(src_name)}`\n") + a(f"- 进程: **{target}**,采样 **{n:,}** 个(1 ms/样本 ≈ **{n/1000:.1f} s CPU**)") + a("- 采集期内各进程采样: " + + "、".join(f"{k} {v:,}" for k, v in agg["all_proc"])) + a(f"- PDB 精确匹配: **{len(matched)}** 个模块" + + (f";版本不一致 {len(mismatched)}: {', '.join(mismatched[:8])}" if mismatched else "") + + (f";有二进制无 PDB {len(no_pdb)}: {', '.join(no_pdb[:8])}" if no_pdb else "")) + if not matched: + a("\n> ⚠️ **没有模块匹配到 PDB**,下面的函数名只有系统 DLL 有,我们的代码只能到「模块 + RVA」。" + "\n> 原因通常是:采集用的二进制不是 `/bin` 下这一份(换了机器构建 / 采完又重编译过)。") + a("") + + def table(title, rows, base=n, fmt="`{}`"): + a(f"## {title}\n") + a("| 采样 | 占比 | 项 |") + a("|---:|---:|---|") + for k, v in rows: + a(f"| {v:,} | {v*100.0/base:.2f}% | {fmt.format(k)} |") + a("") + + table("独占自耗时 by 模块", agg["excl_mod"]) + + a("## 我们模块的函数热点\n") + if agg["our_excl"]: + a("### 自身耗时(exclusive,叶子帧是我们)\n") + a("| 采样 | 占比 | 模块!函数 |") + a("|---:|---:|---|") + for k, v in agg["our_excl"]: + mod, _, fn = k.partition("!") + a(f"| {v:,} | {v*100.0/n:.2f}% | `{short(f'{mod}!{fn}')}` |") + a("") + else: + a("_没有采样落在我们模块里(热点全在被调用的第三方库/系统)_\n") + if agg["our_incl"]: + a("### 出现在调用栈中(inclusive,每栈去重)—— 真正的「骨架函数」\n") + a("| 采样 | 占比 | 模块!函数 |") + a("|---:|---:|---|") + for k, v in agg["our_incl"]: + mod, _, fn = k.partition("!") + a(f"| {v:,} | {v*100.0/n:.2f}% | `{short(f'{mod}!{fn}')}` |") + a("") + if agg["asm_drivers"]: + a("### 驱动 Parasolid(建模内核)的我们的函数\n") + a("| 采样 | 占比 | 模块!函数 |") + a("|---:|---:|---|") + for k, v in agg["asm_drivers"]: + mod, _, fn = k.partition("!") + a(f"| {v:,} | {v*100.0/n:.2f}% | `{short(f'{mod}!{fn}')}` |") + a("") + + a("## 独占自耗时 by 函数\n") + a("| 采样 | 占比 | 模块!函数 / 模块+RVA |") + a("|---:|---:|---|") + for k, v in agg["excl_fn"]: + mod, _, fn = k.partition("!") + a(f"| {v:,} | {v*100.0/n:.2f}% | `{addr_hint(mod, fn, images)}` |") + a("") + + table("栈内出现率(inclusive)", agg["incl_mod"]) + + a("## 线程分布\n") + a("| 采样 | 占比 | 线程 | 该线程主要自耗时模块 |") + a("|---:|---:|---|---|") + for t, v in agg["threads"]: + tops = ", ".join(f"{m}:{c}" for m, c in agg["thread_leaf"].get(t, [])[:3]) + a(f"| {v:,} | {v*100.0/n:.2f}% | tid {t} | {tops} |") + a("") + + a("## 主干调用链(模块级,重复折叠)\n") + for k, v in agg["chains"]: + a(f"- `{v*100.0/n:5.2f}%` ({v:,}) {k}") + a("") + + if agg["sites"]: + a("## 关键调用点(被关注模块的直接调用方)\n") + for label, rows in agg["sites"].items(): + a(f"### {label}\n") + a("| 采样 | 占比 | 调用方 |") + a("|---:|---:|---|") + for k, v in rows: + mod, _, fn = k.partition("!") + a(f"| {v:,} | {v*100.0/n:.2f}% | `{addr_hint(mod, fn, images)}` |") + a("") + + a("## 时间线(每秒采样数 + 主导模块)\n") + a("```") + for sec, total, tops in agg["timeline"]: + if total < 20: + continue + bar = "#" * min(50, int(total / 40)) + desc = " ".join(f"{m.replace('.dll','').replace('.exe','')}:{c}" for m, c in tops) + a(f"t={sec:>4}s {total:>6} {bar} {desc}") + a("```") + + with open(out_path, "w", encoding="utf-8") as f: + f.write("\n".join(L) + "\n") + log(f"[5/5] report: {out_path}") + + +# --------------------------------------------------------------------------- # +def main(): + ap = argparse.ArgumentParser(description="解析 diagsession/ETL 的 CPU 采样热点") + ap.add_argument("file", help=".diagsession 或 .etl") + ap.add_argument("--repo", default=None, + help="仓库根目录(默认从脚本位置推断,用于找 bin/**/*.pdb)") + ap.add_argument("--process", default="acad.exe", help="被测进程名,默认 acad.exe") + ap.add_argument("--out", default=None, help="报告输出路径,默认 perf-report.md") + ap.add_argument("--net-symbols", action="store_true", + help="额外从微软符号服务器拉系统符号(慢,可能几分钟)") + ap.add_argument("--verify-only", action="store_true", help="只校验 PDB 匹配,不做聚合") + args = ap.parse_args() + + src = os.path.abspath(args.file) + if not os.path.exists(src): + raise SystemExit(f"文件不存在: {src}") + + repo = args.repo + if not repo: + guess = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + repo = guess if os.path.isdir(os.path.join(guess, "bin", "Code2026")) else None + log(f"[0/5] repo: {repo or '(not found, skip local PDB)'}") + + workdir = tempfile.mkdtemp(prefix="perf-diag-") + etl = load_etl(src, workdir) + env = build_env(repo, args.net_symbols) + + log("[2/5] reading image list from trace ...") + images = trace_images(etl, env, workdir) + matched, mismatched, no_pdb = verify_symbols(images, repo) + log(f"[3/5] PDB matched: {len(matched)} modules" + + (f", version mismatch: {len(mismatched)}" if mismatched else "")) + ours = [k for k in matched if any(h.lower() in k.lower() for h in OUR_MODULE_HINT)] + log(f" our modules with symbols: {len(ours)} -> {', '.join(sorted(ours)[:10])}" + + (" ..." if len(ours) > 10 else "")) + if mismatched: + log(f" WARNING version mismatch (symbols unusable): " + f"{', '.join(sorted(mismatched)[:10])}") + + if args.verify_only: + log("done (--verify-only).") + return + + log("[3/5] aggregating samples (a 137 s trace takes ~40-90 s, please wait) ...") + agg = aggregate(etl, env, args.process) + + if agg["samples"] == 0: + names = ", ".join(k for k, _ in agg["all_proc"]) or "(none)" + raise SystemExit(f"no samples for '{args.process}'. processes in trace: {names}\n" + f"use --process to pick the right one") + + json_path = os.path.join(workdir, "agg.json") + with open(json_path, "w", encoding="utf-8") as f: + json.dump({k: v for k, v in agg.items()}, f, indent=1, ensure_ascii=False) + log(f"[4/5] raw aggregates: {json_path}") + + out = args.out or os.path.join(os.getcwd(), "perf-report.md") + render(agg, images, (matched, mismatched, no_pdb), src, args.process, out) + + +if __name__ == "__main__": + main()