"""PDB symbol census: Rust vs C++ in Windows binaries (public PDBs). Inputs: raw llvm-pdbutil dumps in ./raw (publics, globals, modules, seccontribs, symbols). Outputs: census TSV, per-PDB crate lists, per-function detail, code-byte breakdown, DWriteCore diff, Narrator math symbols, verification, summary. """ import re, os, sys, bisect, subprocess, ctypes, collections, json from dataclasses import dataclass, field HERE = os.path.dirname(os.path.abspath(__file__)) RAW = os.path.join(HERE, "raw") CXXFILT = r"C:\Program Files\LLVM\bin\llvm-cxxfilt.exe" PDBS = [ # short, label, binary, .text VirtualSize (from llvm-readobj --sections on the installed binary) ("dwritecore161", "DWriteCore.pdb 9DA72605 (Windows App SDK 1.6.1)", r"C:\Windows\SystemApps\Microsoft.WindowsAppRuntime.CBS_8wekyb3d8bbwe\DWriteCore.dll", 0x1CE679), ("dwritecore211", "DWriteCore.pdb 92A5ED42 (Windows App SDK 2.1.1)", r"C:\Windows\SystemApps\Microsoft.WindowsAppRuntime.CBS.2_8wekyb3d8bbwe\DWriteCore.dll", 0x1D9591), ("dwrite", "DWrite.pdb 4E75EC72 (System32 DWrite.dll, negative control)", r"C:\Windows\System32\DWrite.dll", 0x19281C), ("sudo", "sudo.pdb 0ECA8348", r"C:\Windows\System32\sudo.exe", 0x9DEC0), ("udiapi", "udiapi.pdb 400633DF", r"C:\Windows\System32\UdiApiClient.dll", 0x17F569), ("cloudrecovery", "cloudrecoverydownloadtool.pdb 5D5FD7D7", r"C:\Windows\System32\CloudRecoveryDownloadTool.dll", 0x19E9F0), ("narrator", "Narrator.pdb 93804859", r"C:\Windows\System32\Narrator.exe", 0x9FB83), ("srh", "SRH.pdb B5F9A339 (System32 SRH.dll, Narrator screen reader host)", r"C:\Windows\System32\SRH.dll", 0x43B780), ] # ---------------------------------------------------------------- crate classification STD_CRATES = { "core", "alloc", "std", "panic_unwind", "panic_abort", "proc_macro", "compiler_builtins", "hashbrown", "rustc_demangle", "gimli", "addr2line", "miniz_oxide", "object", "memchr", "adler", "adler2", "unwind", "libc", "std_detect", "rustc_std_workspace_core", "rustc_std_workspace_alloc", "rustc_std_workspace_std", "unicode_width", } # Well-known crates.io names (judgment call; anything not listed here or in STD_CRATES is # treated as first-party Microsoft code). THIRD_PARTY_CRATES = { "which", "com", "com_macros", "bumpalo", "serde", "serde_json", "serde_derive", "serde_core", "windows", "windows_core", "windows_result", "windows_strings", "windows_implement", "windows_interface", "windows_sys", "windows_targets", "windows_registry", "windows_version", "windows_numerics", "windows_collections", "windows_future", "windows_link", "windows_threading", "clap", "clap_builder", "clap_lex", "clap_derive", "anstream", "anstyle", "anstyle_parse", "anstyle_query", "anstyle_wincon", "colorchoice", "utf8parse", "is_terminal_polyfill", "once_cell", "lazy_static", "tracing", "tracing_core", "tracing_subscriber", "tracing_attributes", "futures", "futures_util", "futures_core", "futures_channel", "futures_task", "futures_executor", "futures_io", "futures_sink", "futures_macro", "bytes", "indexmap", "equivalent", "regex", "regex_automata", "regex_syntax", "aho_corasick", "phf", "phf_shared", "phf_generator", "roxmltree", "sxd_document", "sxd_xpath", "mathcat", "widestring", "smallvec", "zerocopy", "zerocopy_derive", "foldhash", "bitvec", "funty", "wyz", "radium", "tap", "thiserror", "thiserror_impl", "anyhow", "log", "env_logger", "itertools", "either", "rand", "rand_core", "rand_chacha", "getrandom", "uuid", "chrono", "time", "time_core", "num_traits", "num_integer", "num_bigint", "cfg_if", "static_assertions", "bitflags", "byteorder", "tokio", "tokio_util", "tokio_macros", "mio", "parking_lot", "parking_lot_core", "lock_api", "crossbeam", "crossbeam_utils", "crossbeam_channel", "crossbeam_epoch", "crossbeam_deque", "rayon", "rayon_core", "url", "percent_encoding", "idna", "form_urlencoded", "http", "http_body", "hyper", "reqwest", "base64", "hex", "sha2", "sha1", "digest", "md5", "md_5", "crc32fast", "flate2", "zip", "quick_xml", "xml_rs", "toml", "toml_edit", "ryu", "itoa", "unicode_normalization", "unicode_segmentation", "unicode_bidi", "tinyvec", "tinyvec_macros", "arrayvec", "ahash", "fxhash", "rustc_hash", "nom", "winapi", "ntapi", "winreg", "tracelogging", "tracelogging_dynamic", "win_etw_provider", "win_etw_macros", "win_etw_metadata", "strum", "strum_macros", "derive_more", "num_enum", "scopeguard", "memoffset", "libloading", "semver", "sysinfo", "walkdir", "glob", "tempfile", "dirs", "home", "humantime", "encoding_rs", "fluent", "fluent_bundle", "unic_langid", "intl_memoizer", "fluent_syntax", "type_map", "self_cell", "wide", "half", "bit_set", "bit_vec", "generic_array", "typenum", "block_buffer", "cpufeatures", "ring", "rustls", "webpki", "spin", "untrusted", "pin_project_lite", "pin_project", "pin_utils", "slab", "yoke", "zerofrom", "writeable", "litemap", "tinystr", "displaydoc", "stable_deref_trait", "utf16_iter", "utf8_iter", "allocator_api2", "unicode_ident", "proc_macro2", "quote", "syn", "heck", "textwrap", "strsim", "terminal_size", "is_terminal", "os_str_bytes", "atty", "bstr", "csv", "csv_core", "hashlink", "linked_hash_map", "lru", "dashmap", "arc_swap", "event_listener", "async_channel", "async_trait", "ordered_float", "num_format", "fixed", "num_rational", "num_complex", "num", "num_derive", "wchar", "windows_bindgen", "windows_metadata", "async_std", "smol", "blocking", "socket2", "native_tls", "schannel", "openssl", "hyper_tls", "hyper_util", "h2", "httparse", "httpdate", "mime", "mime_guess", "encoding", "ipnet", "tower", "tower_service", "want", "try_lock", "sync_wrapper", "rustls_pemfile", "webpki_roots", "serde_urlencoded", "serde_with", "erased_serde", "bincode", "postcard", "rmp", "rmp_serde", "ciborium", "prost", "prost_types", "tonic", "sled", "rusqlite", "libsqlite3_sys", "r2d2", "diesel", "xml", "xmlparser", "minidom", "quick_error", "failure", "error_chain", "snafu", "eyre", "color_eyre", "miette", "backtrace", "backtrace_sys", "nix", "errno", "signal_hook", "ctrlc", "num_cpus", "cpu_time", "instant", "wasm_bindgen", "js_sys", "web_sys", "console_error_panic_hook", "indicatif", "console", "dialoguer", "termcolor", "colored", "ansi_term", "owo_colors", "yansi", "nu_ansi_term", "crossterm", "termion", "tui", "ratatui", "cursive", "unicode_truncate", "unicode_linebreak", "hyphenation", "textwrap_macros", "smawk", "fancy_regex", "pcre2", "onig", "globset", "ignore", "same_file", "filetime", "notify", "fsevent", "inotify", "mio_extras", "remove_dir_all", "fs_extra", "fs2", "fs4", "memmap", "memmap2", "mmap", "page_size", "region", "libc_print", "dtoa", "lexical", "lexical_core", "fast_float", "minimal_lexical", "arrayref", "constant_time_eq", "subtle", "zeroize", "secrecy", "sha3", "blake2", "blake3", "keccak", "hmac", "pbkdf2", "argon2", "scrypt", "bcrypt", "aes", "aes_gcm", "chacha20", "chacha20poly1305", "x25519_dalek", "ed25519_dalek", "curve25519_dalek", "p256", "p384", "k256", "ecdsa", "rsa", "signature", "pkcs8", "pkcs1", "der", "spki", "x509_parser", "x509_cert", "asn1_rs", "der_parser", "oid_registry", "rcgen", "rustls_native_certs", "sct", "pem", "pem_rfc7468", "base64ct", "const_oid", "crypto_common", "crypto_bigint", "elliptic_curve", "ff", "group", "rand_xorshift", "rand_pcg", "rand_distr", "rand_hc", "rand_xoshiro", "fastrand", "nanorand", "oorandom", "wyhash", "seahash", "siphasher", "twox_hash", "xxhash_rust", "highway", "metrohash", "fnv", "hash32", "hashbrown_std", "im", "im_rc", "rpds", "persistent", "slotmap", "slab_alloc", "typed_arena", "id_arena", "generational_arena", "thunderdome", "petgraph", "fixedbitset", "roaring", "croaring", "vec_map", "enum_map", "enumset", "smallbitvec", "bit_field", "bitvec_std", "bytemuck", "bytemuck_derive", "safe_transmute", "pod", "plain", "scroll", "goblin", "pelite", "exe", "pdb", "pdb_addr2line", "symbolic", "cpp_demangle", "msvc_demangler", "rustc_demangle_std", "ar", "tar", "xz2", "bzip2", "zstd", "zstd_safe", "lz4", "lz4_flex", "snap", "brotli", "brotli_decompressor", "inflate", "deflate", "libflate", "png", "jpeg_decoder", "image", "imageproc", "resvg", "usvg", "tiny_skia", "ttf_parser", "rustybuzz", "swash", "cosmic_text", "fontdb", "fontdue", "ab_glyph", "ab_glyph_rasterizer", "glyph_brush", "wgpu", "winit", "raw_window_handle", "glow", "glutin", "gl", "vulkano", "ash", "metal", "d3d12", "windows_ext", "com", "com_macros", "wio", "intercom", "vst", "nih_plug", "cpal", "rodio", "hound", "symphonia", "lewton", "minimp3", "claxon", "opus", "audiopus", "webrtc", "str0m", "quinn", "quiche", "s2n_quic", "rustls_pki_types", "aws_lc_rs", "libcrux", "kyber", "dilithium", "num_threads", "threadpool", "scoped_threadpool", "rayon_hash", "executor", "async_executor", "async_io", "async_lock", "async_net", "async_process", "async_task", "concurrent_queue", "polling", "waker_fn", "atomic_waker", "futures_lite", "futures_timer", "async_timer", "timer", "ticker", "governor", "leaky_bucket", "ratelimit", "backoff", "retry", "tokio_retry", "again", "exponential_backoff", "unicode_xid", "unicode_script", "unicode_general_category", "unicode_bidi_mirroring", "unicode_ccc", "unicode_joining_type", "unicode_properties", "unicode_names2", "unicode_case_mapping", "unicode_casing", "caseless", "unidecode", "deunicode", "any_ascii", "icu", "icu_locid", "icu_provider", "icu_collections", "icu_normalizer", "icu_properties", "icu_segmenter", "icu_casemap", "icu_collator", "icu_list", "icu_plurals", "icu_decimal", "icu_datetime", "icu_calendar", "icu_timezone", "fixed_decimal", "zerovec", "zerotrie", "potential_utf", "smallstr", "compact_str", "smartstring", "ecow", "arcstr", "flexstr", "kstring", "istring", "string_cache", "lasso", "interner", "internment", "ustr", "symbol_table", "phf_codegen", "phf_macros", "static_map", "literal", "const_format", "const_str", "konst", "castaway", "downcast_rs", "downcast", "mopa", "as_any", "dyn_clone", "dyn_eq", "dyn_hash", "impls", "auto_impl", "ambassador", "delegate", "enum_dispatch", "enum_delegate", "inventory", "linkme", "ctor", "static_init", "lazy_regex", "once_map", "maplit", "hashbrown_map", "matches", "assert_matches", "pretty_assertions", "similar", "difference", "diff", "dissimilar", "imara_diff", "unified_diff", "patch", "insta", "proptest", "quickcheck", "arbitrary", "libfuzzer_sys", "afl", "honggfuzz", "criterion", "bencher", "iai", "divan", "tango", "mockall", "mockito", "wiremock", "httpmock", "fake", "faker", "dummy", "proptest_derive", "test_case", "rstest", "serial_test", "tempdir", "assert_cmd", "assert_fs", "predicates", "trycmd", "snapbox", "escargot", "duct", "subprocess", "cmd_lib", "xshell", "run_script", "shell_words", "shlex", "clap_complete", "clap_mangen", "structopt", "argh", "gumdrop", "pico_args", "lexopt", "getopts", "docopt", "bpaf", "xflags", "seahorse", "argparse", "dotenv", "dotenvy", "envy", "config", "figment", "toml_datetime", "serde_yaml", "yaml_rust", "ron", "json5", "hjson", "hcl", "kdl", "dhall", "jsonnet", "starlark", "rhai", "rlua", "mlua", "rquickjs", "boa", "deno_core", "v8", "wasmtime", "wasmer", "wasmi", "wasm3", "cranelift", "walrus", "wasmparser", "wat", "wast", "wasm_encoder", "wit_bindgen", "wit_parser", } # ---------------------------------------------------------------- data classes @dataclass class Module: idx: int name: str obj: str cls: str = "" @dataclass class Proc: name: str seg: int off: int size: int modidx: int kind: str # 'L' or 'G' @dataclass class Public: name: str seg: int off: int flags: str mangling: str = "" modidx: int = -1 demangled: str = "" @dataclass class Contrib: sec: str mod: int seg: int off: int size: int code: bool @dataclass class Func: seg: int off: int size: int # -1 unknown (public-only) modidx: int # -1 unknown names: list # all names at this address (procs first) src: str # 'proc', 'public', 'both' lang: str = "" crate: str = "" # rust only mangling: str = "" # ---------------------------------------------------------------- parsing RE_MOD = re.compile(r"^\s*Mod (\d{4}) \| `(.*)`:\s*$") RE_OBJ = re.compile(r"^\s*Obj: `(.*)`:\s*$") RE_PUB = re.compile(r"^\s*(\d+) \| S_PUB32 \[size = (\d+)\] `(.*)`\s*$") RE_PUBADDR = re.compile(r"^\s*flags = ([a-z ,]*), addr = (\d{4}):(\d+)\s*$") RE_PROC = re.compile(r"^\s*(\d+) \| (S_LPROC32|S_GPROC32|S_LPROC32_ID|S_GPROC32_ID|S_THUNK32) \[size = \d+\] `(.*)`\s*$") RE_PROCADDR = re.compile(r"addr = (\d{4}):(\d+), code size = (\d+)") RE_SC = re.compile(r"^\s*SC\[([^\]]*)\]\s*\| mod = (\d+), (\d{4}):(\d+), size = (\d+),") def parse_modules(path): mods = {} cur = None with open(path, encoding="utf-8", errors="replace") as f: for line in f: m = RE_MOD.match(line) if m: cur = Module(int(m.group(1)), m.group(2), "") mods[cur.idx] = cur continue m = RE_OBJ.match(line) if m and cur is not None: cur.obj = m.group(1) return mods def parse_publics(path): pubs = [] pending = None n_pub32_lines = 0 with open(path, encoding="utf-8", errors="replace") as f: for line in f: if "S_PUB32" in line: n_pub32_lines += 1 m = RE_PUB.match(line) if m: pending = m.group(3) continue if pending is not None: m = RE_PUBADDR.match(line) if m: pubs.append(Public(pending, int(m.group(2)), int(m.group(3)), m.group(1).strip())) pending = None else: raise RuntimeError(f"public without addr line: {pending!r} then {line!r}") return pubs, n_pub32_lines def parse_symbols(path): procs = [] curmod = None pending = None with open(path, encoding="utf-8", errors="replace") as f: for line in f: m = RE_MOD.match(line) if m: curmod = int(m.group(1)) continue m = RE_PROC.match(line) if m: pending = (m.group(2), m.group(3)) continue if pending is not None: m = RE_PROCADDR.search(line) if m: kind = "G" if pending[0].startswith("S_GPROC32") else ("T" if pending[0] == "S_THUNK32" else "L") procs.append(Proc(pending[1], int(m.group(1)), int(m.group(2)), int(m.group(3)), curmod, kind)) pending = None return procs def parse_seccontribs(path): contribs = [] cur = None flags = "" with open(path, encoding="utf-8", errors="replace") as f: for line in f: m = RE_SC.match(line) if m: if cur is not None: cur.code = "IMAGE_SCN_CNT_CODE" in flags contribs.append(cur) cur = Contrib(m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), False) flags = "" elif cur is not None: flags += line if cur is not None: cur.code = "IMAGE_SCN_CNT_CODE" in flags contribs.append(cur) return contribs # ---------------------------------------------------------------- classification helpers CRT_HINTS = ["vcstartup", "\\ucrt\\", "libcpmt", "delayimp", "\\crts\\", "osmode_function_map", "msvcrt", "vcruntime", "libcmt", "libucrt", "intermediate\\crt\\", "\\stl\\", "\\crt\\"] def classify_module(m: Module) -> str: n = m.name.lower() o = m.obj.lower() if n.endswith(".rcgu.o") or n.endswith("\\symbols.o") or n.endswith("/symbols.o"): return "rust" if m.name.strip() == "* Linker *" or n.endswith(".exp"): return "linker" if n.endswith(".dll"): return "import" for h in CRT_HINTS: if h in n or h in o: return "crt" if "windows kits" in o: return "sdk" return "native" # includes '* CIL *' (LTCG whole-program C++ module) LANG_LABEL = {"rust": "Rust", "native": "C/C++", "crt": "MSVC CRT/STL", "import": "import thunk", "sdk": "SDK lib", "linker": "linker", "unknown": "unattributed"} def mangling_of(name: str) -> str: if name.startswith("?"): return "msvc" if re.match(r"^_ZN\d", name): return "rust-legacy" if re.match(r"^_R[0-9]*[NCMXYIB][A-Za-z0-9_]", name): return "rust-v0" if name.startswith("__imp_"): return "import" return "plain" # Legacy Rust demangler (fallback / cross-check for llvm-cxxfilt) LEGACY_ESCAPES = [("$SP$", "@"), ("$BP$", "*"), ("$RF$", "&"), ("$LT$", "<"), ("$GT$", ">"), ("$LP$", "("), ("$RP$", ")"), ("$C$", ",")] def legacy_unescape(seg: str) -> str: if seg.startswith("_$"): seg = seg[1:] out = [] i = 0 while i < len(seg): if seg.startswith("..", i): out.append("::"); i += 2; continue if seg[i] == "$": j = seg.find("$", i + 1) if j > i: tok = seg[i:j + 1] rep = None for k, v in LEGACY_ESCAPES: if tok == k: rep = v if rep is None and tok.startswith("$u") and re.match(r"^\$u[0-9a-f]+\$$", tok): rep = chr(int(tok[2:-1], 16)) if rep is not None: out.append(rep); i = j + 1; continue out.append(seg[i]); i += 1 return "".join(out) def demangle_legacy_py(sym: str): m = re.match(r"^_ZN(.*?)E(\.[A-Za-z0-9_.]+)?$", sym) if not m: return None body = m.group(1) segs = [] i = 0 while i < len(body): j = i while j < len(body) and body[j].isdigit(): j += 1 if j == i: return None n = int(body[i:j]); seg = body[j:j + n]; i = j + n if len(seg) != n: return None segs.append(seg) hash_ = None if segs and re.match(r"^h[0-9a-f]{16}$", segs[-1]): hash_ = segs.pop() return "::".join(legacy_unescape(s) for s in segs) def cxxfilt(names): if not names: return {} p = subprocess.run([CXXFILT, "-n"], input="\n".join(names) + "\n", capture_output=True, text=True) out = p.stdout.splitlines() res = {} for a, b in zip(names, out): res[a] = b return res _dbghelp = None def undname(name: str, flags=0x1000) -> str: global _dbghelp if _dbghelp is None: _dbghelp = ctypes.WinDLL("dbghelp") _dbghelp.UnDecorateSymbolName.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint, ctypes.c_uint] _dbghelp.UnDecorateSymbolName.restype = ctypes.c_uint buf = ctypes.create_string_buffer(32768) r = _dbghelp.UnDecorateSymbolName(name.encode("utf-8", "replace"), buf, 32768, flags) if r: return buf.value.decode("utf-8", "replace") return name def strip_hash(demangled: str) -> str: return re.sub(r"::h[0-9a-f]{16}$", "", demangled) def _match_angle(s, i): """s[i] == '<'; return index of matching '>' or -1.""" depth = 0 while i < len(s): if s[i] == "<": depth += 1 elif s[i] == ">": depth -= 1 if depth == 0: return i i += 1 return -1 def crate_root(name: str): """Root crate of a Rust path in rustc's MSVC debuginfo style or rustc-demangle style. Rules: the first path segment at angle-bracket depth 0. If that segment is an MSVC type wrapper (enum2$, ref$, ptr_const$, tuple$<..>, ...), descend into T. If the name starts with '<' (demangled `::f` / `::f`), use the implementing type T. Returns None when no `crate::` path is present (e.g. #[no_mangle] extern "C" names). """ s = name.strip() if not s: return None if s.startswith("<"): j = _match_angle(s, 0) inner = s[1:j] if j > 0 else s[1:] if " as " in inner: inner = inner.split(" as ", 1)[0] elif inner.startswith("impl ") and " for " in inner: inner = inner.split(" for ", 1)[1] return crate_root(inner) depth = 0 i = 0 while i < len(s): c = s[i] if c == "<": depth += 1 elif c == ">": depth -= 1 elif depth == 0 and s.startswith("::", i): seg = s[:i] if "$<" in seg or seg.endswith(">"): k = seg.find("<") inner = seg[k + 1:seg.rfind(">")] if seg.endswith(">") else seg[k + 1:] return crate_root(inner) if " " in seg or "(" in seg or "*" in seg or "&" in seg: return None return seg i += 1 return None def crate_class(c: str) -> str: if c in STD_CRATES: return "stdlib" if c in THIRD_PARTY_CRATES: return "third-party" return "first-party" # ---------------------------------------------------------------- per PDB analysis class PdbData: def __init__(self, short, label, binary, text_vsize): self.short, self.label, self.binary, self.text_vsize = short, label, binary, text_vsize self.mods = parse_modules(os.path.join(RAW, f"{short}.modules.txt")) for m in self.mods.values(): m.cls = classify_module(m) self.pubs, self.n_pub32_lines = parse_publics(os.path.join(RAW, f"{short}.publics.txt")) self.procs = parse_symbols(os.path.join(RAW, f"{short}.symbols.txt")) self.contribs = parse_seccontribs(os.path.join(RAW, f"{short}.seccontribs.txt")) self._index_contribs() self._attribute_publics() self._demangle_publics() self._build_funcs() self._code_bytes() # address -> module via section contributions def _index_contribs(self): self.by_seg = collections.defaultdict(list) for c in self.contribs: self.by_seg[c.seg].append(c) self.seg_starts = {} for seg, lst in self.by_seg.items(): lst.sort(key=lambda c: c.off) self.seg_starts[seg] = [c.off for c in lst] def contrib_at(self, seg, off): lst = self.by_seg.get(seg) if not lst: return None i = bisect.bisect_right(self.seg_starts[seg], off) - 1 if i >= 0: c = lst[i] if c.off <= off < c.off + max(c.size, 1): return c return None def _attribute_publics(self): for p in self.pubs: p.mangling = mangling_of(p.name) c = self.contrib_at(p.seg, p.off) p.modidx = c.mod if c else -1 def _demangle_publics(self): rust = [p for p in self.pubs if p.mangling in ("rust-legacy", "rust-v0")] res = cxxfilt(sorted({p.name for p in rust})) for p in rust: d = res.get(p.name, p.name) if d == p.name and p.mangling == "rust-legacy": d = demangle_legacy_py(p.name) or p.name p.demangled = strip_hash(d) def lang_of_mod(self, modidx): if modidx is None or modidx < 0: return "unknown" m = self.mods.get(modidx) return m.cls if m else "unknown" def _build_funcs(self): byaddr = {} for pr in self.procs: if pr.kind == "T": continue key = (pr.seg, pr.off) f = byaddr.get(key) if f is None: f = Func(pr.seg, pr.off, pr.size, pr.modidx, [pr.name], "proc") byaddr[key] = f else: f.names.append(pr.name) if f.size < 0: f.size = pr.size if f.modidx < 0: f.modidx = pr.modidx for p in self.pubs: if "function" not in p.flags: continue key = (p.seg, p.off) f = byaddr.get(key) if f is None: f = Func(p.seg, p.off, -1, p.modidx, [p.name], "public") byaddr[key] = f else: f.names.append(p.name) if f.src == "proc": f.src = "both" if f.mangling == "" and p.mangling in ("msvc", "rust-legacy", "rust-v0"): f.mangling = p.mangling if p.demangled and not any("::" in n for n in f.names[:-1]): f.names.append("[demangled] " + p.demangled) self.funcs = list(byaddr.values()) # cross-check: a proc's module vs the contribution owning its address self.proc_mod_mismatch = 0 self.proc_mod_checked = 0 for pr in self.procs: c = self.contrib_at(pr.seg, pr.off) if c is not None: self.proc_mod_checked += 1 if c.mod != pr.modidx: self.proc_mod_mismatch += 1 for f in self.funcs: f.lang = self.lang_of_mod(f.modidx) if f.lang == "unknown": # fall back to mangling when address lookup failed mg = [mangling_of(n) for n in f.names if not n.startswith("[demangled]")] if any(m.startswith("rust") for m in mg): f.lang = "rust" elif any(m == "msvc" for m in mg): f.lang = "native" if f.lang == "rust": f.crate = self.rust_crate_for(f) or "(no Rust path: extern/no_mangle)" def rust_crate_for(self, f: Func): for n in f.names: if n.startswith("[demangled] "): n = n[len("[demangled] "):] mg = mangling_of(n) if mg == "rust-legacy": d = demangle_legacy_py(n) if d: r = crate_root(d) if r: return r elif mg == "rust-v0": d = cxxfilt([n]).get(n, n) r = crate_root(strip_hash(d)) if r: return r else: r = crate_root(n) if r: return r return None def _code_bytes(self): self.code_by_mod = collections.Counter() self.code_by_cls = collections.Counter() self.code_total = 0 for c in self.contribs: if c.code: self.code_by_mod[c.mod] += c.size self.code_by_cls[self.lang_of_mod(c.mod)] += c.size self.code_total += c.size # ---- helpers for reports def all_names(self): for p in self.pubs: yield ("public", p.name, p.demangled, p) for pr in self.procs: yield ("proc", pr.name, "", pr) def rust_publics(self): return [p for p in self.pubs if p.mangling in ("rust-legacy", "rust-v0")] def rust_crate_stats(self): """crate -> dict(funcs, bytes_known, bytes)""" st = collections.defaultdict(lambda: {"funcs": 0, "bytes": 0, "sized": 0}) for f in self.funcs: if f.lang != "rust": continue d = st[f.crate] d["funcs"] += 1 if f.size >= 0: d["bytes"] += f.size d["sized"] += 1 # Rust data publics (statics) grouped too, counted separately data = collections.Counter() for p in self.rust_publics(): if "function" in p.flags: continue data[crate_root(p.demangled) or "(no path)"] += 1 return st, data def coverage(self): """class -> (sized function bytes, contributed code bytes, pct string)""" sized = collections.Counter() for f in self.funcs: if f.size >= 0: sized[f.lang] += f.size return {cls: (sized[cls], self.code_by_cls[cls], pct(sized[cls], self.code_by_cls[cls])) for cls in ("rust", "native", "crt")} def mismatch_crosstab(self): ct = collections.Counter() for pr in self.procs: c = self.contrib_at(pr.seg, pr.off) if c is not None and c.mod != pr.modidx: ct[(self.lang_of_mod(pr.modidx), self.lang_of_mod(c.mod))] += 1 return ct def func_counts(self): c = collections.Counter(f.lang for f in self.funcs) funclets = collections.Counter() for f in self.funcs: if any(re.search(r"::(dtor|catch|fin|filt)\$\d+$", n) for n in f.names): funclets[f.lang] += 1 return c, funclets # ---------------------------------------------------------------- reports def pct(a, b): return f"{(100.0 * a / b):.1f}%" if b else "n/a" def fmt_int(n): return f"{n:,}" def write(path, text): with open(os.path.join(HERE, path), "w", encoding="utf-8", newline="\n") as f: f.write(text) # ---------------------------------------------------------------- Narrator / SRH / NarratorMCAT READOBJ = r"C:\Program Files\LLVM\bin\llvm-readobj.exe" MCAT_DLL = r"C:\Windows\System32\NarratorMCAT\NarratorMCAT.dll" MATH_PATS = ["MathCAT", "MCAT", "mathcat", "MathML", "Math", "Equation", "OMath"] MATH_MODULE_RE = r"mathprocessor|mathutils|gesturemath|mathcat|mcat" def readobj(args): try: return subprocess.run([READOBJ] + args, capture_output=True, text=True, errors="replace").stdout except OSError as e: return "" def reading_of(name, dem=""): return undname(name) if name.startswith("?") else (dem or name) def modname(d, idx): m = d.mods.get(idx) return m.name.split("\\")[-1] if m else str(idx) def math_hits_for(d): rows = []; seen = set() for kind, name, dem, obj in d.all_names(): hit = [p for p in MATH_PATS if p in name] or [p for p in MATH_PATS if dem and p in dem] if not hit and re.search(r"math|mcat|equation", name, re.I): hit = ["(case-insensitive)"] if not hit: continue key = (kind, name) if key in seen: continue seen.add(key) reading = reading_of(name, dem) full = undname(name, 0) if name.startswith("?") else "" addr = f"{obj.seg:04d}:{obj.off}" if kind == "public": extra = obj.flags + f", module {modname(d, obj.modidx)}" else: extra = f"code size {obj.size}, module {modname(d, obj.modidx)}" rows.append((reading, kind, addr, extra, name, full, ",".join(hit))) rows.sort() return rows def mcat_exports(): ex = re.findall(r"^\s*Name: (\S+)", readobj(["--coff-exports", MCAT_DLL]), re.M) return ex or ["SetRulesDir", "SetMathML", "GetSpokenText", "SetPreference", "GetPreference", "GetBraille", "DoNavigateKeyPress", "DoNavigateCommand", "GetNavigationMathML", "GetNavigationMathMLId", "GetMathCATVersion", "FreeMathCATString"] def math_section(d, title): N = [title, "=" * len(title), ""] rows = math_hits_for(d) npub = len(d.pubs); nproc = len(d.procs); nmod = len(d.mods) N.append(f"Symbols whose name (mangled or demangled) contains MathCAT / MCAT / mathcat / MathML / Math / Equation / OMath") N.append(f"(searched {npub:,} publics, {nproc:,} module S_*PROC32 records and {nmod} module names; a case-insensitive net for math|mcat|equation is applied too)") N.append(f"matches: {len(rows)}") N.append("") if not rows: N.append(f"RESULT: none. Not one of the {npub:,} publics, {nproc:,} local-function records or {nmod} module names in {d.label}") N.append("contains 'math', 'mcat' or 'equation' in any letter case. No MathCAT wrapper, MathML handling or MathCAT C-API") N.append("caller is identifiable in this binary by name. A name search cannot establish that none exists.") N.append("") for reading, kind, addr, extra, name, full, hit in rows: N.append(f"[{kind}] {addr} {extra} matched: {hit}") N.append(f" verbatim : {name}") N.append(f" reading : {reading}") if full and full != reading: N.append(f" full : {full}") # module-owned symbols for math-named object files owned = [m for m in sorted(d.mods.values(), key=lambda m: m.idx) if re.search(MATH_MODULE_RE, m.name, re.I) and m.cls == "native"] if owned: N.append("") N.append("Object files with 'math' in their name, and every symbol the linker attributes to them (by section contribution):") for m in owned: procs = [pr for pr in d.procs if pr.modidx == m.idx] pubs = [p for p in d.pubs if p.modidx == m.idx] N.append(f" module {m.idx:04d} {m.name} (code bytes {d.code_by_mod.get(m.idx, 0):,}; {len(pubs)} publics, {len(procs)} local procs)") for p in pubs: N.append(f" [public] {reading_of(p.name)} <- {p.name}") for pr in procs: N.append(f" [proc] {pr.name} (code size {pr.size})") # LoadLibrary / Dll names N.append(""); N.append("-" * 78) ll = [] seen = set() for kind, name, dem, obj in d.all_names(): if ("LoadLibrary" in name or "Dll" in name) and (kind, name) not in seen: seen.add((kind, name)); ll.append((kind, name)) N.append(f"Symbols containing LoadLibrary or Dll (case-sensitive): {len(ll)}") for kind, name in ll: r = reading_of(name) N.append(f" [{kind}] {name}" + (f" reading: {r}" if r != name else "")) # MathCAT C-API export names (whole-word) N.append(""); N.append("-" * 78) ex = mcat_exports() N.append("MathCAT C-API export names of NarratorMCAT.dll (" + ", ".join(ex) + ") as whole words anywhere in this PDB's names:") found = [] for kind, name, dem, obj in d.all_names(): r = reading_of(name, dem) for c in ex: if re.search(r"(? {r} (contains {c})") N.extend(found or [" (none)"]) # activation-style names N.append(""); N.append("-" * 78) N.append("Activation-style names (CoCreateInstance / RoActivateInstance / RoGetActivationFactory / GetProcAddress / GetActivationFactory / CLSID_*Math / IID_*Math):") found = [] for kind, name, dem, obj in d.all_names(): if re.search(r"CoCreateInstance|RoActivateInstance|RoGetActivationFactory|GetProcAddress|GetActivationFactory|CLSID_.*Math|IID_.*Math", name): found.append(f" [{kind}] {name} -> {reading_of(name, dem)}") N.extend(found or [" (none)"]) # import modules and first-party object files imps = sorted({m.name.split("Import:")[-1] for m in d.mods.values() if m.name.lower().endswith(".dll")}, key=str.lower) N.append(""); N.append(f"Import modules in the PDB ({len(imps)}): " + ", ".join(imps)) fp = sorted({m.name.split("\\")[-1] for m in d.mods.values() if m.cls == "native"}, key=str.lower) N.append(""); N.append(f"First-party object files in the PDB ({len(fp)}):") N.append(" " + ", ".join(fp)) return N def pe_strings_report(path, pats): """ASCII + UTF-16LE strings in a PE file matching any of pats (case-insensitive).""" try: data = open(path, "rb").read() except OSError as e: return [f" {path}: {e}"] rx = re.compile(b"(?i)" + b"|".join(re.escape(p.encode()) for p in pats) + b"|" + b"|".join(re.escape(p.encode("utf-16le")) for p in pats)) out = [] seen = set() for m in rx.finditer(data): s = m.group() enc = "utf16" if b"\x00" in s else "ascii" # widen to the surrounding printable run if enc == "ascii": a = m.start() while a > 0 and 0x20 <= data[a - 1] < 0x7f: a -= 1 b = m.end() while b < len(data) and 0x20 <= data[b] < 0x7f: b += 1 txt = data[a:b].decode("ascii", "replace") else: a = m.start() while a >= 2 and 0x20 <= data[a - 2] < 0x7f and data[a - 1] == 0: a -= 2 b = m.end() while b + 1 < len(data) and 0x20 <= data[b] < 0x7f and data[b + 1] == 0: b += 2 txt = data[a:b].decode("utf-16le", "replace") key = (enc, txt) if key in seen: continue seen.add(key) out.append(f" 0x{a:08x} [{enc}] {txt[:160]}") return out or [" (none)"] def dll_name_strings(path): """Distinct *.dll names present as UTF-16LE strings (the form LoadLibraryW callers embed).""" try: data = open(path, "rb").read() except OSError: return [] names = set() for m in re.finditer(rb"(?:[\x20-\x7e]\x00){3,}", data): s = m.group().decode("utf-16le") for x in re.findall(r"[A-Za-z0-9_.\-]+\.(?:dll|DLL|Dll)", s): names.add(x) return sorted(names, key=str.lower) def narratormcat_facts(): L = ["NarratorMCAT.dll (the MathCAT DLL) facts, read from the file on disk", "-" * 78] if not os.path.exists(MCAT_DLL): L.append(f"{MCAT_DLL}: not present on this machine") return L L.append(f"path: {MCAT_DLL} ({os.path.getsize(MCAT_DLL):,} bytes)") dbg = readobj(["--coff-debug-directory", MCAT_DLL]) g = re.search(r"PDBGUID: (\{[^}]+\})", dbg); ag = re.search(r"PDBAge: (\d+)", dbg); pn = re.search(r"PDBFileName: (\S+)", dbg) L.append(f"CodeView record: PDB {pn.group(1) if pn else '?'} GUID {g.group(1) if g else '?'} age {ag.group(1) if ag else '?'}") L.append(" ('libmathcat_c-.pdb' is cargo's output name for MathCAT's C-API crate; symchk against msdl.microsoft.com: not found, so no PDB census is possible for it)") ex = re.findall(r"^\s*Name: (\S+)", readobj(["--coff-exports", MCAT_DLL]), re.M) L.append(f"exports ({len(ex)}): " + ", ".join(ex)) im = re.findall(r"^\s*Name: (\S+)", readobj(["--coff-imports", MCAT_DLL]), re.M) L.append(f"imports ({len(im)}): " + ", ".join(im)) data = open(MCAT_DLL, "rb").read() std_paths = sorted({m.group().decode() for m in re.finditer(rb"library[\\/](?:std|core|alloc)[\\/]src[\\/][A-Za-z0-9_\\/.]+\.rs", data)}) crate_paths = sorted({m.group().decode() for m in re.finditer(rb"[A-Za-z0-9_\-]+-[0-9]+\.[0-9]+\.[0-9]+[\\/]src[\\/][A-Za-z0-9_\\/.]+\.rs", data)}) L.append(f"Rust markers inside the DLL: {len(std_paths)} distinct Rust std/core/alloc source paths (e.g. {std_paths[:3]}),") L.append(f" {len(crate_paths)} distinct '-/src/*.rs' paths; crates named: " + ", ".join(sorted({p.split('/')[0].split(chr(92))[0] for p in crate_paths})[:60])) rev = os.path.join(os.path.dirname(MCAT_DLL), "REVISION.txt") if os.path.exists(rev): L.append("REVISION.txt next to it: " + " / ".join(open(rev, encoding="utf-8", errors="replace").read().split())) rules = os.path.join(os.path.dirname(MCAT_DLL), "Rules") if os.path.isdir(rules): L.append("Rules directory next to it: " + ", ".join(sorted(os.listdir(rules)))) L.append("") L.append("Strings in Narrator.exe matching narratormcat|mathcat|mcat|srh.dll|SetRulesDir|GetSpokenText (ASCII or UTF-16):") L.extend(pe_strings_report(r"C:\Windows\System32\Narrator.exe", ["narratormcat", "mathcat", "mcat", "srh.dll", "SetRulesDir", "GetSpokenText"])) L.append("Strings in SRH.dll matching the same patterns:") L.extend(pe_strings_report(r"C:\Windows\System32\SRH.dll", ["narratormcat", "mathcat", "mcat", "SetRulesDir", "GetSpokenText", "SetMathML"])) L.append("") for lab, p in (("Narrator.exe", r"C:\Windows\System32\Narrator.exe"), ("SRH.dll", r"C:\Windows\System32\SRH.dll")): n = dll_name_strings(p) L.append(f"All *.dll names stored as UTF-16 strings in {lab} ({len(n)}), i.e. the names a LoadLibraryW-style loader would embed:") L.append(" " + ", ".join(n)) L.append(" SRH.dll's own plug-in loader (DllLoader::LoadDll + SignedDllValidator, used by BrailleSubsystem::Init) names liblouis.dll and") L.append(" brlapi.dll this way, so a NarratorMCAT.dll loaded by the same mechanism would be visible here. It is not.") L.append("") ls = os.path.join(HERE, "loader-search.txt") if os.path.exists(ls): L.append("Disk-wide loader search (findloader.py):") L.extend(" " + x for x in open(ls, encoding="utf-8").read().rstrip().splitlines()) else: L.append("(loader-search.txt not present; the disk-wide scan is reported in narrator-mathcat-static.txt)") return L def math_reports(data): out = [] out.extend(math_section(data["narrator"], "Narrator.pdb (Narrator.exe) math-related symbols")) out.append(""); out.append("") if "srh" in data: out.extend(math_section(data["srh"], "SRH.pdb (SRH.dll, Narrator's screen reader host, loaded by Narrator.exe at run time) math-related symbols")) out.append(""); out.append("") out.extend(narratormcat_facts()) return out def main(): data = {} for short, label, binary, vs in PDBS: print("parsing", short, file=sys.stderr) data[short] = PdbData(short, label, binary, vs) # ---------------- census TSV cols = ["pdb", "binary", "publics_total", "publics_rust_mangled", "publics_msvc", "publics_plain_or_other", "rust_pct_of_publics", "publics_rust_function", "publics_rust_data", "named_functions_total", "functions_rust", "functions_cpp_native", "functions_crt_stl", "functions_other", "rust_pct_of_functions", "cpp_eh_funclets_counted", "code_bytes_total_attributed", "code_bytes_rust", "code_bytes_cpp_native", "code_bytes_crt_stl", "code_bytes_other", "rust_pct_of_code_bytes", "text_section_vsize", "rust_fn_records_cover_pct_of_rust_code", "cpp_fn_records_cover_pct_of_cpp_code", "function_count_reliability", "crates_stdlib", "crates_third_party", "crates_first_party", "top5_first_party_crates"] rows = [] for short, *_ in PDBS: d = data[short] pubs = d.pubs n = len(pubs) rust_m = [p for p in pubs if p.mangling.startswith("rust")] msvc = sum(1 for p in pubs if p.mangling == "msvc") plain = n - len(rust_m) - msvc rust_fn = sum(1 for p in rust_m if "function" in p.flags) fc, funclets = d.func_counts() ftotal = len(d.funcs) other_f = ftotal - fc["rust"] - fc["native"] - fc["crt"] cb = d.code_by_cls other_cb = d.code_total - cb["rust"] - cb["native"] - cb["crt"] st, _ = d.rust_crate_stats() real = {c: v for c, v in st.items() if not c.startswith("(")} ncls = collections.Counter(crate_class(c) for c in real) top5 = sorted(((v["funcs"], c) for c, v in real.items() if crate_class(c) == "first-party"), reverse=True)[:5] cov = d.coverage() rcov = cov["rust"][2]; ccov = cov["native"][2] if cb["rust"] == 0: rel = "no Rust code in binary" elif cov["rust"][0] * 100 < cov["rust"][1] * 50: rel = "UNRELIABLE for Rust: first-party crates compiled without debuginfo, only std/alloc/core functions have records; use code bytes" else: rel = "usable: Rust function records cover most Rust code bytes; C++ counted from publics + local records" rows.append([short, os.path.basename(d.binary), n, len(rust_m), msvc, plain, pct(len(rust_m), n), rust_fn, len(rust_m) - rust_fn, ftotal, fc["rust"], fc["native"], fc["crt"], other_f, pct(fc["rust"], ftotal), funclets["native"] + funclets["crt"], d.code_total, cb["rust"], cb["native"], cb["crt"], other_cb, pct(cb["rust"], d.code_total), d.text_vsize, rcov, ccov, rel, ncls["stdlib"], ncls["third-party"], ncls["first-party"], ", ".join(f"{c} ({k})" for k, c in top5)]) tsv = "\t".join(cols) + "\n" + "\n".join("\t".join(str(x) for x in r) for r in rows) + "\n" write("pdb-symbol-census.tsv", tsv) # ---------------- per-PDB crate lists + function detail + code bytes for short, *_ in PDBS: d = data[short] st, datastat = d.rust_crate_stats() lines = [f"# Rust crates in {d.label}", f"# binary: {d.binary}", "# Source: S_LPROC32/S_GPROC32 records of rustc .rcgu.o modules + publics attributed to those modules by address.", "# 'functions' = distinct code addresses; 'code_bytes' = sum of 'code size' of those with a size record (sized_functions).", "# class: stdlib = Rust standard library and its vendored deps; third-party = well-known crates.io name (judgment);", "# first-party = everything else (assumed Microsoft-authored).", ] rmods = [m.name for m in d.mods.values() if m.cls == "rust"] lines.append("# rustc object modules in this PDB (the first path segment before the hash is the ROOT crate cargo built): " + ("; ".join(rmods) if rmods else "none")) cov = d.coverage()["rust"] if d.code_by_cls["rust"] and cov[0] * 100 < cov[1] * 50: lines.append(f"# WARNING: sized Rust function records cover only {cov[2]} of the Rust code bytes ({cov[0]:,} of {cov[1]:,}). The root crate and its") lines.append("# first-party dependencies were compiled without debuginfo, so only std/alloc/core functions (which ship with debuginfo)") lines.append("# have records. First-party crate names surface only through Rust-mangled DATA publics (ETW metadata), listed below.") lines.append("# Do not read the function counts below as the crate composition of this binary; use code bytes.") lines.append("crate\tclass\tfunctions\tsized_functions\tcode_bytes") for c, v in sorted(st.items(), key=lambda kv: (-kv[1]["funcs"], kv[0])): cls = crate_class(c) if not c.startswith("(") else "n/a" lines.append(f"{c}\t{cls}\t{v['funcs']}\t{v['sized']}\t{v['bytes']}") if datastat: lines.append("") lines.append("# Rust-mangled DATA publics (statics such as ETW EVENT_METADATA), by crate:") for c, k in datastat.most_common(): lines.append(f"{c}\t{crate_class(c)}\t{k}\t(data)") tot = collections.Counter(); totb = collections.Counter() for c, v in st.items(): cls = crate_class(c) if not c.startswith("(") else "extern-c/no_mangle" tot[cls] += v["funcs"]; totb[cls] += v["bytes"] lines.append("") lines.append("# Totals by class: functions / code bytes") for cls in ("first-party", "third-party", "stdlib", "extern-c/no_mangle"): lines.append(f"{cls}\t{tot[cls]}\t{totb[cls]}") write(f"crates-{short}.txt", "\n".join(lines) + "\n") # function detail fl = ["seg:off\tsize\tlang\tmodule\tcrate\tsrc\tnames"] for f in sorted(d.funcs, key=lambda f: (f.seg, f.off)): mn = d.mods[f.modidx].name if f.modidx in d.mods else "?" fl.append(f"{f.seg:04d}:{f.off}\t{f.size}\t{f.lang}\t{mn}\t{f.crate}\t{f.src}\t" + " || ".join(f.names)) write(f"functions-{short}.tsv", "\n".join(fl) + "\n") # code bytes cl = [f"# Code bytes by contributing object file, {d.label}", "# Source: llvm-pdbutil dump -section-contribs, contributions flagged IMAGE_SCN_CNT_CODE, summed per module.", f"# .text VirtualSize of installed binary = {d.text_vsize} bytes; attributed code contributions sum = {d.code_total} ({pct(d.code_total, d.text_vsize)} of .text; remainder is alignment padding / linker-generated).", "", "class\tcode_bytes\tpct_of_attributed"] for cls, b in d.code_by_cls.most_common(): cl.append(f"{LANG_LABEL.get(cls, cls)}\t{b}\t{pct(b, d.code_total)}") cl.append(""); cl.append("module_idx\tclass\tcode_bytes\tmodule") for mi, b in d.code_by_mod.most_common(): m = d.mods.get(mi) cl.append(f"{mi}\t{m.cls if m else '?'}\t{b}\t{m.name if m else '?'}") write(f"code-bytes-{short}.txt", "\n".join(cl) + "\n") # ---------------- DWriteCore diff a, b = data["dwritecore161"], data["dwritecore211"] def side(d): fc, funclets = d.func_counts() st, _ = d.rust_crate_stats() return fc, funclets, st fa, fla, sta = side(a); fb, flb, stb = side(b) L = ["DWriteCore: Windows App SDK 1.6.1 (PDB 9DA72605) -> 2.1.1 (PDB 92A5ED42)", "=" * 78, ""] L.append("Method: functions = distinct code addresses named by S_LPROC32/S_GPROC32 module records or by") L.append("publics with flags=function; language = the object file that contributed the address (rustc") L.append(".rcgu.o = Rust; hand-written .obj = C/C++; MSVC CRT/STL objects listed separately). Code bytes =") L.append("section contributions flagged as code, per contributing object. Public PDBs omit inlined code") L.append("and every count is a lower bound; ratios by count are not ratios by bytes.") L.append("") def row(name, x, y): return f"{name:<44}{x:>12,}{y:>12,}{(y - x):>+12,}{(' (' + pct(y - x, x) + ')') if x else ''}" L.append(f"{'':<44}{'1.6.1':>12}{'2.1.1':>12}{'delta':>12}") L.append(row("Rust functions (by object)", fa["rust"], fb["rust"])) L.append(row("C/C++ functions (by object, incl. EH funclets)", fa["native"], fb["native"])) L.append(row(" of which EH funclets (dtor$/catch$)", fla["native"], flb["native"])) L.append(row(" C/C++ functions excluding funclets", fa["native"] - fla["native"], fb["native"] - flb["native"])) L.append(row("MSVC CRT/STL functions", fa["crt"], fb["crt"])) L.append(row("All named functions", len(a.funcs), len(b.funcs))) L.append("") L.append(row("Rust code bytes", a.code_by_cls["rust"], b.code_by_cls["rust"])) L.append(row("C/C++ code bytes", a.code_by_cls["native"], b.code_by_cls["native"])) L.append(row("MSVC CRT/STL code bytes", a.code_by_cls["crt"], b.code_by_cls["crt"])) L.append(row("All attributed code bytes", a.code_total, b.code_total)) L.append(row(".text VirtualSize (binary)", a.text_vsize, b.text_vsize)) L.append("") L.append(f"Rust share of named functions: {pct(fa['rust'], len(a.funcs))} -> {pct(fb['rust'], len(b.funcs))}") L.append(f"Rust share of attributed code bytes: {pct(a.code_by_cls['rust'], a.code_total)} -> {pct(b.code_by_cls['rust'], b.code_total)}") L.append("") L.append("Publics by mangling (for completeness; Rust functions are LTO-internalized and never reach the publics table):") for d in (a, b): n = len(d.pubs); r = sum(1 for p in d.pubs if p.mangling.startswith("rust")); m = sum(1 for p in d.pubs if p.mangling == "msvc") L.append(f" {d.short}: publics={n:,} rust-mangled={r} msvc={m:,} plain/other={n - r - m:,}") L.append("") cova, covb = a.coverage(), b.coverage() L.append(f"Coverage of Rust code bytes by sized function records: {cova['rust'][2]} (1.6.1), {covb['rust'][2]} (2.1.1);") L.append(f"C/C++: {cova['native'][2]} and {covb['native'][2]} (external C++ functions are named by publics but carry no size in a public PDB).") L.append("") _f = lambda st, c: st.get(c, {"funcs": 0})["funcs"] L.append(f"Naming note: the umbrella crate that implements the DirectWrite API surface is called 'api_impl' in 1.6.1") L.append(f"({_f(sta, 'api_impl')} functions) and 'dwrite' in 2.1.1 ({_f(stb, 'dwrite')} functions); 'api_impl' survives in 2.1.1 only as a small crate of") L.append(f"{_f(stb, 'api_impl')} functions. Treat api_impl+dwrite as one unit when comparing. Likewise the COM plumbing moved from the") L.append(f"'com' crate (1.6.1, {_f(sta, 'com')} fns) to 'windows_core' (2.1.1, {_f(stb, 'windows_core')} fns); both are Microsoft-authored crates published on crates.io.") ua = sta.get("api_impl", {"funcs": 0})["funcs"] + sta.get("dwrite", {"funcs": 0})["funcs"] ub = stb.get("api_impl", {"funcs": 0})["funcs"] + stb.get("dwrite", {"funcs": 0})["funcs"] L.append(f" api_impl + dwrite combined: {ua} -> {ub} functions ({ub - ua:+})") L.append("") ca = {c for c in sta if not c.startswith("(")}; cbs = {c for c in stb if not c.startswith("(")} def crate_line(c, x, y): return f" {c:<32}{crate_class(c):<13}{x:>8}{y:>8}{(y - x):>+8}" L.append("Crates added in 2.1.1 (not present in 1.6.1):") for c in sorted(cbs - ca): L.append(crate_line(c, 0, stb[c]["funcs"])) if not (cbs - ca): L.append(" (none)") L.append("Crates removed in 2.1.1 (present in 1.6.1 only):") for c in sorted(ca - cbs): L.append(crate_line(c, sta[c]["funcs"], 0)) if not (ca - cbs): L.append(" (none)") L.append("") L.append("Per-crate function count, every crate (first-party listed first, then third-party, then stdlib):") L.append(f" {'crate':<32}{'class':<13}{'1.6.1':>8}{'2.1.1':>8}{'delta':>8}") allc = sorted(ca | cbs, key=lambda c: ({"first-party": 0, "third-party": 1, "stdlib": 2}[crate_class(c)], -(stb.get(c, {"funcs": 0})["funcs"]), c)) for c in allc: L.append(crate_line(c, sta.get(c, {"funcs": 0})["funcs"], stb.get(c, {"funcs": 0})["funcs"])) L.append("") L.append("Per-crate Rust code bytes (sized functions only):") L.append(f" {'crate':<32}{'class':<13}{'1.6.1':>10}{'2.1.1':>10}{'delta':>10}") for c in allc: x = sta.get(c, {"bytes": 0})["bytes"]; y = stb.get(c, {"bytes": 0})["bytes"] L.append(f" {c:<32}{crate_class(c):<13}{x:>10,}{y:>10,}{(y - x):>+10,}") # plain-language verdict L.append("") dr = fb["rust"] - fa["rust"]; dc = fb["native"] - fa["native"]; dcx = (fb["native"] - flb["native"]) - (fa["native"] - fla["native"]) dbr = b.code_by_cls["rust"] - a.code_by_cls["rust"]; dbc = b.code_by_cls["native"] - a.code_by_cls["native"] def word(x): return "grew" if x > 0 else "shrank" if x < 0 else "was unchanged" L.append("Verdict (plain language):") L.append(f" By code bytes (the reliable measure): the Rust side {word(dbr)} by {abs(dbr):,} bytes ({pct(dbr, a.code_by_cls['rust'])}); the C/C++ side {word(dbc)} by {abs(dbc):,} bytes ({pct(dbc, a.code_by_cls['native'])}).") L.append(f" By named-function count: Rust {word(dr)} by {abs(dr):,} ({pct(dr, fa['rust'])}); C/C++ {word(dc)} by {abs(dc):,} ({pct(dc, fa['native'])}; {abs(dcx):,} excluding EH funclets).") L.append(f" Rust share of code bytes {pct(a.code_by_cls['rust'], a.code_total)} -> {pct(b.code_by_cls['rust'], b.code_total)}; share of named functions {pct(fa['rust'], len(a.funcs))} -> {pct(fb['rust'], len(b.funcs))}.") L.append(" So: the Rust side did not shrink and the C++ side did not shrink either; both grew by a few percent in bytes and the") L.append(" Rust share stayed flat. The lower function counts come from fewer, larger functions (inlining / different rustc and MSVC") L.append(" versions), not from code being removed. No Rust crate replaced a C++ component between 1.6.1 and 2.1.1; the visible") L.append(" changes are renames (api_impl -> dwrite, com -> windows_core) and the new shaping_cache crate.") # C++ component breakdown for DWriteCore (CMake target dirs) L.append("") L.append("C/C++ code bytes by CMake target (module name prefix before '.dir\\'), 1.6.1 vs 2.1.1:") def comp_bytes(d): cc = collections.Counter() for mi, bts in d.code_by_mod.items(): m = d.mods.get(mi) if not m or m.cls != "native": continue nm = m.name k = nm.split(".dir\\")[0].split("\\")[-1] if ".dir\\" in nm else os.path.basename(nm) cc[k] += bts return cc cca, ccb = comp_bytes(a), comp_bytes(b) L.append(f" {'target':<36}{'1.6.1':>10}{'2.1.1':>10}{'delta':>10}") for k in sorted(set(cca) | set(ccb), key=lambda k: -ccb.get(k, 0)): L.append(f" {k:<36}{cca.get(k, 0):>10,}{ccb.get(k, 0):>10,}{(ccb.get(k, 0) - cca.get(k, 0)):>+10,}") write("dwritecore-symbol-diff.txt", "\n".join(L) + "\n") # ---------------- DWrite negative control d = data["dwrite"] rm = [p for p in d.pubs if p.mangling.startswith("rust")] rust_procs = [pr for pr in d.procs if d.lang_of_mod(pr.modidx) == "rust"] rustish = [pr.name for pr in d.procs if crate_root(pr.name) in STD_CRATES | {"windows", "windows_core"}] rcgu = [m.name for m in d.mods.values() if m.cls == "rust"] T = [f"DWrite.pdb (classic DirectWrite, System32 DWrite.dll) negative control", "", f"publics total: {len(d.pubs):,} (functions {sum(1 for p in d.pubs if 'function' in p.flags):,}, data {sum(1 for p in d.pubs if 'function' not in p.flags):,})", f"Rust-mangled publics (_ZN... or _R...): {len(rm)}", f"rustc object modules (*.rcgu.o / symbols.o) in module list: {len(rcgu)} (modules total {len(d.mods)})", f"S_*PROC32 records inside rustc modules: {len(rust_procs)}", f"module-record names whose root path is a Rust std crate (core/alloc/std...): {len(rustish)}", f"named functions total: {len(d.funcs):,}; code bytes attributed: {d.code_total:,} of .text {d.text_vsize:,}", "", "Conclusion: no Rust of any kind is visible in DWrite.pdb by mangling, by object-file origin, or by path-style names."] write("dwrite-negative-control.txt", "\n".join(T) + "\n") # ---------------- Narrator / SRH math symbols + NarratorMCAT facts write("narrator-symbols-math.txt", "\n".join(math_reports(data)) + "\n") # ---------------- verification V = ["Verification", "=" * 78, ""] V.append("1. Rust demangler demonstration (llvm-cxxfilt 21.1.4 -n; Python legacy demangler as cross-check)") dwc_rust = [] for short in ("dwritecore161", "dwritecore211"): for p in data[short].rust_publics(): dwc_rust.append((short, p.name)) for short, nm in dwc_rust: d1 = cxxfilt([nm]).get(nm, nm); d2 = demangle_legacy_py(nm) V.append(f" [{short}] {nm}") V.append(f" llvm-cxxfilt : {d1}") V.append(f" python-legacy: {d2} {'(agree after hash strip)' if d2 == strip_hash(d1) else '(DIFFER)'}") V.append(f" DWriteCore has only {len(dwc_rust)} Rust-mangled publics in total (both PDBs); all other Rust functions are LTO-internalized") V.append(" and appear as S_LPROC32 records already in rustc's readable MSVC debuginfo form. Three such names from 2.1.1 module 0:") for pr in [x for x in data["dwritecore211"].procs if x.modidx == 0][:3]: V.append(f" {pr.name} -> crate root: {crate_root(pr.name)}") V.append(" Third mangled sample (v0 scheme) from sudo.pdb:") sp = data["sudo"].rust_publics()[0] V.append(f" {sp.name}") V.append(f" llvm-cxxfilt : {cxxfilt([sp.name]).get(sp.name)} -> crate root: {crate_root(sp.demangled)}") V.append(" Synthetic legacy check for the python demangler:") syn = "_ZN70_$LT$alloc..vec..Vec$LT$T$GT$$u20$as$u20$core..ops..drop..Drop$GT$4drop17h0123456789abcdefE" body = "_$LT$alloc..vec..Vec$LT$T$GT$$u20$as$u20$core..ops..drop..Drop$GT$" syn = f"_ZN{len(body)}{body}4drop17h0123456789abcdefE" V.append(f" {syn}") V.append(f" python-legacy: {demangle_legacy_py(syn)} -> crate root: {crate_root(demangle_legacy_py(syn))}") V.append(f" llvm-cxxfilt : {cxxfilt([syn]).get(syn)}") V.append("") V.append("2. Parsed publics count vs grep -c S_PUB32 on the raw dump") for short, *_ in PDBS: d = data[short] V.append(f" {short:<14} parsed={len(d.pubs):>6} S_PUB32 lines in raw dump={d.n_pub32_lines:>6} {'OK' if len(d.pubs) == d.n_pub32_lines else 'MISMATCH'}") V.append("") V.append("3. Sanity: font_face present in both DWriteCore PDBs. The region/rgn matches below are") V.append(" DWriteCore's own C++ font-region classes, not the kernel driver's Rust rgncore crate.") for short in ("dwritecore161", "dwritecore211", "dwrite"): d = data[short] names = [n for _, n, _, _ in d.all_names()] ff = [n for n in names if "font_face" in n] rg = [n for n in names if re.search(r"region|rgn", n, re.I)] V.append(f" {short}: names containing 'font_face' = {len(ff)}; e.g. {ff[0] if ff else '-'}") V.append(f" {short}: names matching /region|rgn/i = {len(rg)}" + (f"; e.g. {rg[:3]}" if rg else "")) V.append("") V.append("4. Module attribution cross-check: for every S_*PROC32 record, the section contribution covering its address") V.append(" must belong to the same module (checks that seccontrib 'mod' is 0-based like the module list).") for short, *_ in PDBS: d = data[short] ct = d.mismatch_crosstab() cross = ", ".join(f"{a}->{b}: {n}" for (a, b), n in sorted(ct.items(), key=lambda kv: -kv[1])) or "-" V.append(f" {short:<14} procs checked={d.proc_mod_checked:>6} module mismatches={d.proc_mod_mismatch:>6} by class (proc's module -> contributing module): {cross}") V.append(" Mismatches are ICF/COMDAT folding and LTCG placement; none crosses the Rust/C++ line, so language attribution is unaffected.") V.append("") V.append("5. Mangling vs object-file language cross-check on publics (MSVC-mangled names should never land in rustc objects,") V.append(" Rust-mangled names never in C++ objects; identical-code folding can legitimately merge a few).") for short, *_ in PDBS: d = data[short] m_in_r = [p.name for p in d.pubs if p.mangling == "msvc" and d.lang_of_mod(p.modidx) == "rust"] r_in_c = [p.name for p in d.pubs if p.mangling.startswith("rust") and d.lang_of_mod(p.modidx) not in ("rust", "unknown")] unattr = sum(1 for p in d.pubs if p.modidx < 0) V.append(f" {short:<14} msvc-in-rust-obj={len(m_in_r)} rust-in-cpp-obj={len(r_in_c)} publics-unattributed-by-address={unattr}") for nm in m_in_r[:5]: V.append(f" msvc name in rust object: {nm}") for nm in r_in_c[:5]: V.append(f" rust name in c++ object: {nm}") V.append("") V.append("6. Crate-root parser garbage check (roots containing '$', '<', hex-only, or empty):") bad = 0 for short, *_ in PDBS: st, _ = data[short].rust_crate_stats() for c in st: if c.startswith("("): continue if "$" in c or "<" in c or ">" in c or re.fullmatch(r"[0-9a-f]+", c) or not c: bad += 1 V.append(f" {short}: suspicious crate root {c!r}") if not bad: V.append(" none found") V.append("") V.append("7. Publics attributed by address to rustc objects but not Rust-mangled (the extern \"C\" bridge surface), per PDB:") for short, *_ in PDBS: d = data[short] br = [p for p in d.pubs if d.lang_of_mod(p.modidx) == "rust" and not p.mangling.startswith("rust")] fn = [p for p in br if "function" in p.flags] V.append(f" {short:<14} total={len(br)} functions={len(fn)} data={len(br) - len(fn)}") for p in fn[:60]: V.append(f" fn {p.name}") write("verification.txt", "\n".join(V) + "\n") # ---------------- summary S = ["PDB symbol census: Rust vs C++ in eight Windows binaries (public PDBs)", "=" * 78, ""] S.append("Tooling: llvm-pdbutil 21.1.4 (dump -publics/-globals/-modules/-symbols/-section-contribs), llvm-cxxfilt 21.1.4") S.append("for Rust legacy/v0 names, dbghelp UnDecorateSymbolName for MSVC readings. All PDB GUIDs match the CodeView") S.append("records of the installed binaries (checked with llvm-readobj).") S.append("") S.append("How to read the numbers") S.append("- Public PDBs carry public (external) symbols plus, in these eight, S_LPROC32 records for local functions with") S.append(" their code sizes, but no inlined functions, no types, no line info. Every count is a lower bound.") S.append("- Rust in these binaries is compiled with LTO into ONE codegen unit (one *.rcgu.o module). Nearly all Rust") S.append(" functions are therefore internal symbols: they never reach the publics table and are NOT _ZN/_R mangled") S.append(" in the PDB. Counting Rust by mangled publics alone finds almost nothing (DWriteCore 2.1.1: 0 of 8,424).") S.append(" In these LTO'd user-mode binaries the only Rust-mangled publics are #[link_section] statics (ETW") S.append(" EVENT_METADATA, TLS callbacks). This does NOT generalise: win32kbase_rs.sys has 177 Rust-v0-mangled") S.append(" function publics (see driver/symbols-win32kbase_rs.txt).") S.append("- Language is therefore attributed by contributing OBJECT FILE: the linker's section-contribution table maps") S.append(" every attributed code byte and the symbol addresses it covers to the module (.rcgu.o = rustc; .obj = MSVC C/C++; CRT/STL and") S.append(" import-thunk objects separated). This gives both a function count and a code-byte comparison.") S.append("- 'Functions' = distinct named code addresses. MSVC emits EH funclets (dtor$N, catch$N) as separate local") S.append(" functions, which inflates C++ counts; they are reported so they can be subtracted.") S.append("- Rust crate attribution uses the root path segment of rustc's readable MSVC debuginfo names (impl blocks") S.append(" appear under the crate that contains the impl; std generics instantiated for first-party types count under std).") S.append("") S.append("Headline numbers") for short, *_ in PDBS: d = data[short] fc, funclets = d.func_counts() n = len(d.pubs); rm = sum(1 for p in d.pubs if p.mangling.startswith("rust")); ms = sum(1 for p in d.pubs if p.mangling == "msvc") st, _ = d.rust_crate_stats() real = {c: v for c, v in st.items() if not c.startswith("(")} ncls = collections.Counter(crate_class(c) for c in real) fp = sorted(((v["funcs"], c) for c, v in real.items() if crate_class(c) == "first-party"), reverse=True) S.append(f"* {os.path.basename(d.binary)} [{d.label}]") S.append(f" publics {n:,}: Rust-mangled {rm} ({pct(rm, n)}), MSVC C++ {ms:,} ({pct(ms, n)}), plain/other {n - rm - ms:,}") S.append(f" named functions {len(d.funcs):,}: Rust {fc['rust']:,} ({pct(fc['rust'], len(d.funcs))}), C/C++ {fc['native']:,} ({pct(fc['native'], len(d.funcs))}; {funclets['native']:,} are EH funclets), CRT/STL {fc['crt']:,}, other {len(d.funcs) - fc['rust'] - fc['native'] - fc['crt']:,}") cb = d.code_by_cls S.append(f" code bytes {d.code_total:,} (of .text {d.text_vsize:,}): Rust {cb['rust']:,} ({pct(cb['rust'], d.code_total)}), C/C++ {cb['native']:,} ({pct(cb['native'], d.code_total)}), CRT/STL {cb['crt']:,} ({pct(cb['crt'], d.code_total)}), other {d.code_total - cb['rust'] - cb['native'] - cb['crt']:,}") if real: S.append(f" Rust crates: {len(real)} (stdlib {ncls['stdlib']}, third-party {ncls['third-party']}, first-party {ncls['first-party']}); top first-party: " + ", ".join(f"{c} {k}" for k, c in fp[:6])) S.append("") S.append("Narrator and MathCAT (see narrator-symbols-math.txt)") nar = data["narrator"]; srh = data.get("srh") S.append(f"- Narrator.pdb: {len(nar.pubs):,} publics, {len(nar.procs):,} local-function records, 0 Rust; {len(math_hits_for(nar))} names mention math/MathCAT/MathML.") if srh: S.append(f"- SRH.pdb (SRH.dll, the screen reader host that Narrator.exe loads at run time; the string 'SRH.DLL' is in Narrator.exe): {len(srh.pubs):,} publics, {len(srh.procs):,} local-function records, 0 Rust;") S.append(f" {len(math_hits_for(srh))} names mention math (MathUtils::IsAtMathBoundary, Math::Sign, a MathBoundaryCheck test) but none mention MathCAT, MathML or the MathCAT C API.") S.append("- NarratorMCAT.dll lives in C:\\Windows\\System32\\NarratorMCAT\\ (not System32 itself). Its CodeView record names libmathcat_c-.pdb, i.e. it is") S.append(" MathCAT's own C-API crate compiled by cargo, and it carries Rust std source paths; its exports are exactly the MathCAT C API") S.append(" (SetRulesDir, SetMathML, GetSpokenText, GetBraille, DoNavigateKeyPress, ...). Its PDB is not on the public symbol server.") srh_dlls = dll_name_strings(r"C:\Windows\System32\SRH.dll") S.append("- Neither Narrator.exe nor SRH.dll contains the strings 'NarratorMCAT', 'MathCAT' or any MathCAT export name. SRH.dll DOES embed") S.append(" the names of the plug-in DLLs it loads through DllLoader::LoadDll + SignedDllValidator (" + ", ".join(x for x in srh_dlls if x.lower() in ("liblouis.dll", "brlapi.dll", "microsoft.cognitiveservices.speech.core.dll", "magnification.dll")) + "),") S.append(" so a NarratorMCAT.dll loaded the same way would show up; it does not. The disk-wide byte scan found no other PE file naming") S.append(" NarratorMCAT or MathCAT; the only export-name hits outside the DLL are four SetMathML strings in WinUIEdit.dll and") S.append(" Microsoft.UI.Text.winmd, which are RichEdit's own math API and unrelated (see narrator-mathcat-static.txt). So the public") S.append(" symbols and strings do NOT reveal the loading") S.append(" path in this build; they do not establish a specific LoadLibrary/GetProcAddress bridge. What the PDBs do") S.append(" show: Narrator.exe -> SRH.dll (string 'SRH.DLL' in Narrator.exe, LoadLibraryExW import); SRH.dll has MathUtils::IsAtMathBoundary") S.append(" (UIA text-range math boundary detection behind WIL feature flags) and the empty mathprocessor.obj; MathCAT itself is a separate") S.append(" Rust DLL (MathCAT 0.7.0 / MathCATForC 0.7.0 per REVISION.txt) with its rules directory beside it.") S.append("") S.append("See dwritecore-symbol-diff.txt, dwrite-negative-control.txt, narrator-symbols-math.txt, verification.txt,") S.append("crates-.txt, code-bytes-.txt and functions-.tsv for detail.") write("summary.txt", "\n".join(S) + "\n") print("done", file=sys.stderr) if __name__ == "__main__": main()