diff --git a/scripts/macos_bundle_dylibs.py b/scripts/macos_bundle_dylibs.py index b6deb2d1ae32..2a1e22fb0520 100644 --- a/scripts/macos_bundle_dylibs.py +++ b/scripts/macos_bundle_dylibs.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 -"""Bundle Homebrew dylib dependencies into a MeshLib.framework version dir. +"""Bundle dylib dependencies into a MeshLib.framework or a macOS .app. -Walks all Mach-O files under /bin and /lib, copies any -dependency that resolves under a Homebrew prefix into /lib, and -rewrites the LC_LOAD_DYLIB / LC_ID_DYLIB entries to @rpath/. An -LC_RPATH is added so the bundled libs are found relative to the binary: - - executables in bin/ get @executable_path/../lib - - dylibs in lib/ (incl. just-bundled ones) get @loader_path/. +Walks the layout's seed directories, copies any dependency resolving under a +Homebrew prefix or a --search-dir into its destination directory, and rewrites +LC_LOAD_DYLIB / LC_ID_DYLIB to @rpath/. Each binary gets an LC_RPATH +pointing at that destination, computed from where it sits: + + framework bin/ -> @executable_path/../lib + lib/ -> @loader_path/. + app Contents/MacOS/ -> @executable_path/../Frameworks + Contents/Frameworks/ -> @loader_path/. + Contents/Frameworks/meshlib/ -> @loader_path/.. + +A .app is assembled from a build tree rather than an install tree, so it needs +--search-dir for the build and prebuilt thirdparty directories. System libraries (/usr/lib, /System) and libpython* are intentionally left as external references. @@ -16,7 +23,7 @@ on every dep in requirements/macos.txt. Why a script rather than dylibbundler / CMake BundleUtilities -(`fixup_bundle`): +(`fixup_bundle`), for the framework and for the .app alike: - fixup_bundle's containment check requires every bundled item's filesystem path to be string-prefixed by the bundle's "dotapp_dir" @@ -39,7 +46,13 @@ - dylibbundler 1.0.5 (the version Homebrew ships) hardcodes /usr/local and /opt/homebrew as the only search prefixes; the arm64 self-hosted build runner installs Homebrew at /Users/runner/.homebrew. This - script calls `brew --prefix` at startup. + script calls `brew --prefix` at startup. It also drops absolute + LC_RPATH entries while resolving @rpath references, which is where + its "can't get path for '@rpath/...'" warnings come from. + - `fixup_bundle` copies every prerequisite it can resolve and keys them + by file name, so a bundle referencing Python through more than one + path ends up shipping a second interpreter. SKIP_BASENAME_RE below is + this script's answer to the same problem. - Primitive install_name_tool / otool calls are made via delocate.tools (already a build-time dep used by the NuGet-patch pipeline). The remaining bespoke code is the algorithm: BFS over @@ -51,6 +64,7 @@ from __future__ import annotations import argparse +import os import re import shutil import stat @@ -97,6 +111,8 @@ def _detect_homebrew_prefixes() -> tuple[str, ...]: HOMEBREW_PREFIXES = _detect_homebrew_prefixes() +# From --search-dir; empty for the framework. +SEARCH_DIRS: tuple[Path, ...] = () SYSTEM_PREFIXES = ("/usr/lib/", "/System/") RELATIVE_PREFIXES = ("@rpath/", "@loader_path/", "@executable_path/") # Leave these to the host system / Homebrew @@ -135,7 +151,9 @@ def should_bundle(load_path: str) -> bool: return False if load_path.startswith(SYSTEM_PREFIXES): return False - if not load_path.startswith(HOMEBREW_PREFIXES): + if not load_path.startswith(HOMEBREW_PREFIXES) and not any( + load_path.startswith(f"{d}/") for d in map(str, SEARCH_DIRS) + ): return False if SKIP_BASENAME_RE.match(Path(load_path).name): return False @@ -158,13 +176,18 @@ def codesign_adhoc(p: Path) -> None: ]) -def _resolve_homebrew_basename(name: str) -> Path | None: - """Find a dylib by basename in any Homebrew lib dir (cached glob).""" +def _resolve_by_basename(name: str) -> Path | None: + """Find a dylib by basename in a --search-dir or Homebrew lib dir.""" if SKIP_BASENAME_RE.match(name): return None - cached = _resolve_homebrew_basename._cache # type: ignore[attr-defined] + cached = _resolve_by_basename._cache # type: ignore[attr-defined] if name in cached: return cached[name] + for extra in SEARCH_DIRS: + cand = extra / name + if cand.exists(): + cached[name] = cand.resolve() + return cached[name] for pref in HOMEBREW_PREFIXES: for sub in ("lib", "opt/*/lib", "Cellar/*/*/lib"): for cand in Path(pref).glob(f"{sub}/{name}"): @@ -180,15 +203,32 @@ def _resolve_homebrew_basename(name: str) -> Path | None: return None -_resolve_homebrew_basename._cache = {} # type: ignore[attr-defined] +_resolve_by_basename._cache = {} # type: ignore[attr-defined] -def bundle(framework_dir: Path) -> None: - bin_dir = framework_dir / "bin" - lib_dir = framework_dir / "lib" +def _rpath_for(p: Path, dest_dir: Path, exe_dirs: list[Path]) -> str: + """Where this binary should look for dest_dir, relative to itself. + + @loader_path for the dylibs, so they resolve whichever process loads them. + """ + anchor = "@executable_path" if any( + p.is_relative_to(d) for d in exe_dirs + ) else "@loader_path" + rel = Path(os.path.relpath(dest_dir, p.parent)).as_posix() + return f"{anchor}/{rel}" + + +def bundle( + seed_dirs: list[Path], + dest_dir: Path, + exe_dirs: list[Path], + sign_exes: bool = True, + drop_absolute_rpaths: bool = False, +) -> None: + lib_dir = dest_dir lib_dir.mkdir(parents=True, exist_ok=True) - seeds = collect_machos(bin_dir) + collect_machos(lib_dir) + seeds = [m for d in seed_dirs for m in collect_machos(d)] log(f"seed mach-o files: {len(seeds)}") # Pre-index Mach-O basenames already present in the framework's lib tree @@ -241,7 +281,7 @@ def bundle_from(src: Path, name: str | None = None) -> Path | None: # Original link-time path is gone from the current bottle # (the very drift we're guarding against). Fall back to # the basename under the standard Homebrew lib dir. - fallback = _resolve_homebrew_basename(req_name) + fallback = _resolve_by_basename(req_name) if fallback is None: log(f"WARN: cannot resolve {dep}; skipping") continue @@ -266,7 +306,7 @@ def bundle_from(src: Path, name: str | None = None) -> Path | None: if cand.exists(): target = bundle_from(cand.resolve(), name) if target is None: - fb = _resolve_homebrew_basename(name) + fb = _resolve_by_basename(name) if fb is not None: target = bundle_from(fb, name) if target is not None and target not in visited: @@ -276,7 +316,7 @@ def bundle_from(src: Path, name: str | None = None) -> Path | None: # delocate.tools.* helpers wrap install_name_tool and ad-hoc sign after # each call; the final codesign_adhoc preserves entitlements/flags that # delocate's default signing would drop. - all_files = collect_machos(bin_dir) + collect_machos(lib_dir) + all_files = [m for d in seed_dirs for m in collect_machos(d)] for p in all_files: make_writable(p) sp = str(p) @@ -291,24 +331,77 @@ def bundle_from(src: Path, name: str | None = None) -> Path | None: sp, dep, f"@rpath/{Path(dep).name}", ad_hoc_sign=False, ) - rpath = "@executable_path/../lib" if p.is_relative_to(bin_dir) else "@loader_path/." + # dyld searches these on the user's machine, and they leak build paths. + if drop_absolute_rpaths: + for rp in get_rpaths(sp): + if not rp.startswith("@"): + log(f"drop rpath {rp} from {p.name}") + subprocess.check_call( + ["install_name_tool", "-delete_rpath", rp, sp], + ) + + rpath = _rpath_for(p, lib_dir, exe_dirs) if rpath not in get_rpaths(sp): add_rpath(sp, rpath, ad_hoc_sign=False) - codesign_adhoc(p) + # Signing a bundle's main executable signs the whole bundle and fails + # on nested items it does not consider code; those callers sign the + # bundle as a unit themselves. + if sign_exes or not any(p.is_relative_to(d) for d in exe_dirs): + codesign_adhoc(p) log(f"bundled {len(bundled)} dylibs into {lib_dir}") +def bundle_framework(framework_dir: Path) -> None: + bundle( + seed_dirs=[framework_dir / "bin", framework_dir / "lib"], + dest_dir=framework_dir / "lib", + exe_dirs=[framework_dir / "bin"], + ) + + +def bundle_app(app_dir: Path) -> None: + contents = app_dir / "Contents" + bundle( + seed_dirs=[contents / "MacOS", contents / "Frameworks"], + dest_dir=contents / "Frameworks", + exe_dirs=[contents / "MacOS"], + sign_exes=False, + drop_absolute_rpaths=True, + ) + + def main() -> None: + global SEARCH_DIRS ap = argparse.ArgumentParser(description=__doc__) ap.add_argument( - "framework_version_dir", + "root", + type=Path, + help="MeshLib.framework/Versions/, or a .app directory", + ) + ap.add_argument( + "--layout", + choices=("framework", "app"), + default="framework", + help="Bundle layout of `root` (default: framework)", + ) + ap.add_argument( + "--search-dir", type=Path, - help="Path to MeshLib.framework/Versions/", + action="append", + default=[], + help="Extra directory to bundle libraries from; repeatable", ) args = ap.parse_args() - bundle(args.framework_version_dir.resolve()) + SEARCH_DIRS = tuple(d.resolve() for d in args.search_dir if d.is_dir()) + if SEARCH_DIRS: + log(f"search dirs: {', '.join(map(str, SEARCH_DIRS))}") + root = args.root.resolve() + if args.layout == "app": + bundle_app(root) + else: + bundle_framework(root) if __name__ == "__main__":