From 1294ab893abf84e1f9e2d351dffe570cc6e4f157 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Mon, 3 Aug 2026 05:26:31 -0500 Subject: [PATCH] [rocKE] Add rocKE COD test driver Add run_rocke.sh, a standalone driver that runs one rocKE lane against the compiler-of-the-day and prints canonical ROCKE_RESULT rows plus a short summary, together with the modules it executes: the result line format, the comgr/toolchain probe, the COD interop smoke driver and the JUnit converter. Every row carries a relevance tier, so triage can tell a genuine compiler-of-the-day regression from a rocKE logic failure or this harness's own plumbing. rocke_relevance.py is a pytest plugin that records, per test, what evidence it saw; the JUnit converter joins that onto the rows, and the lanes without per-test evidence tag their rows from the lane's own class. A skip is scored on what is missing: this host (no GPU, no ROCm torch) keeps it green and unmeasured, while a reason naming the toolchain is a red row, since that is the compiler under test failing to do the work. The pytest lane provisions the test dependencies rocKE itself declares and builds its C++ engine extension with the COD through rocKE's own ROCKE_BUILD_PYBIND option, so the cross-engine tests run against a COD-built engine instead of skipping for a missing module. It lives here, beside the other run_*.sh drivers, so an engineer can check rocKE against a COD without the openmp-ci tree; the nightly drives the same script through its USER-run-rocKE-* wrappers, as CK's driver already does. AI-assisted. --- bin/rocke/README | 15 + bin/rocke/rocke_cod_probe.py | 40 + bin/rocke/rocke_cod_smoke.py | 326 ++++++++ bin/rocke/rocke_junit_results.py | 191 +++++ bin/rocke/rocke_relevance.py | 238 ++++++ bin/rocke/rocke_result.py | 65 ++ bin/run_rocke.sh | 1183 ++++++++++++++++++++++++++++++ 7 files changed, 2058 insertions(+) create mode 100644 bin/rocke/README create mode 100644 bin/rocke/rocke_cod_probe.py create mode 100755 bin/rocke/rocke_cod_smoke.py create mode 100755 bin/rocke/rocke_junit_results.py create mode 100644 bin/rocke/rocke_relevance.py create mode 100644 bin/rocke/rocke_result.py create mode 100755 bin/run_rocke.sh diff --git a/bin/rocke/README b/bin/rocke/README new file mode 100644 index 000000000..3a66668da --- /dev/null +++ b/bin/rocke/README @@ -0,0 +1,15 @@ +Worker modules for ../run_rocke.sh, the rocKE compiler-of-the-day test driver. + +- rocke_result.py the ROCKE_RESULT line format every lane emits +- rocke_relevance.py pytest plugin: can a red row be a COD regression? +- rocke_cod_probe.py which comgr rocKE would load, for the hygiene gate +- rocke_cod_smoke.py COD interop smoke lanes (codegen / comgr / occupancy) +- rocke_junit_results.py JUnit XML -> result rows, joined with the relevance + manifest + +The rocke_ prefix stays: these go on the PYTHONPATH of rocKE's own pytest +session, where a plain result.py would shadow the project's module. + +The CI-side extractor and the full documentation live in the apps repo under +openmp-ci/rocKE (extract-rocke.sh, README.md); the extractor imports +rocke_result.py and rocke_relevance.py from here so the format cannot drift. diff --git a/bin/rocke/rocke_cod_probe.py b/bin/rocke/rocke_cod_probe.py new file mode 100644 index 000000000..0598212e6 --- /dev/null +++ b/bin/rocke/rocke_cod_probe.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Report the comgr library rocKE will actually load, so run_rocke.sh's hygiene +# gate can prove it resolves inside the COD. Prints one line: +# +# +# +# ir-flavor is what the loaded comgr's ROCm vintage implies (>= 7.2 -> llvm22, +# else llvm20); interface-version comes from the lib's own amd_comgr_get_version +# (vintage-proof). Any field is "?"/"UNRESOLVED" when it cannot be determined. + +from __future__ import annotations + +import ctypes + +try: + from rocke.runtime.comgr import resolved_lib_path, resolved_lib_rocm_version + + path = resolved_lib_path() or "UNRESOLVED" + ver = resolved_lib_rocm_version() + rocm = f"{ver[0]}.{ver[1]}" if ver else "?" + # No vintage means no basis for a flavor; claiming llvm20 would make + # run_rocke.sh warn about a mismatch it never measured. + flavor = "?" if ver is None else ("llvm22" if ver >= (7, 2) else "llvm20") +except Exception: + path, rocm, flavor = "UNRESOLVED", "?", "?" + +iface = "?" +try: + fn = ctypes.CDLL(path).amd_comgr_get_version + fn.argtypes = [ctypes.POINTER(ctypes.c_size_t)] * 2 + major, minor = ctypes.c_size_t(), ctypes.c_size_t() + fn(ctypes.byref(major), ctypes.byref(minor)) + iface = f"{major.value}.{minor.value}" +except Exception: + pass + +print(flavor, rocm, iface, path) diff --git a/bin/rocke/rocke_cod_smoke.py b/bin/rocke/rocke_cod_smoke.py new file mode 100755 index 000000000..b3ca09d0e --- /dev/null +++ b/bin/rocke/rocke_cod_smoke.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# rocKE <-> compiler-of-the-day (COD) interop smoke for a single GPU arch. +# +# Modes, all feeding rocKE-issued IR to the COD (see README's lane table): +# codegen : -> COD `clang` -> amdgcn relocatable object +# comgr : -> COD `libamd_comgr` -> HSACO, plus an on-device load+symbol +# check when the local device matches the target arch +# occupancy : -> HSACO, then its codegen resource footprint from the ELF notes +# via rocke.benchmark.perf.occupancy; a spill is reported red +# +# One arch per process, so a fatal LLVM/comgr abort costs that arch's row rather +# than the whole lane. Result lines go to stdout, diagnostics to stderr, and the +# traceback only under ROCKE_DEBUG; always exits 0 so a failure is a data row. + +from __future__ import annotations + +import argparse +import atexit +import os +import shutil +import subprocess +import sys +import tempfile +import traceback +from pathlib import Path +from typing import Any + +from rocke_relevance import TIER_COMPILER +from rocke_result import emit as _result_emit + +GROUP = "universal_gemm" +_OCCUPANCY_GROUP = "occupancy" +_SUBTEST_SUFFIX = "" + + +def _emit(subtest: str, status: int, message: str = "") -> None: + # Every row here comes from driving the COD toolchain, so a red one is + # always a candidate compiler regression. + _result_emit(GROUP, subtest + _SUBTEST_SUFFIX, status, message, TIER_COMPILER) + + +def _maybe_traceback() -> None: + # The row already carries the concise reason, so this is debug-only noise. + if os.environ.get("ROCKE_DEBUG"): + traceback.print_exc() + + +def _is_rdna(arch: str) -> bool: + """RDNA (gfx11xx/gfx12xx) lowers through WMMA at wave32; CDNA is wave64.""" + return arch.startswith(("gfx11", "gfx12")) + + +def _comgr_flavor(requested: str) -> str: + """Resolve `auto` only from the COD clang flavor validated by the worker.""" + if requested and requested != "auto": + return requested + + flavor = os.environ.get("ROCKE_COD_CLANG_FLAVOR") + if flavor not in {"llvm20", "llvm22"}: + raise RuntimeError( + "auto comgr flavor requires ROCKE_COD_CLANG_FLAVOR from the COD worker" + ) + return flavor + + +def _lower_ir(arch: str, flavor: str) -> tuple[str, str]: + """Lower the fixed smoke GEMM to IR, with the symbol the HSACO exports. + + rocKE mangles the spec into the kernel name, and the mangling is + arch-dependent, so the device-load probe has to ask for the built name. + """ + from rocke.instances.common.gemm_universal import ( + DataSpec, + TileSpec, + TraitSpec, + UniversalGemmSpec, + build_universal_gemm, + ) + from rocke.core.lower_llvm import lower_kernel_to_llvm + + is_rdna = _is_rdna(arch) + # compv3 is a CDNA/MFMA-only pipeline. + pipeline = "wmma_v1" if is_rdna else "compv3" + wave_size = 32 if is_rdna else 64 + warp_m, warp_n, warp_k = 2, 2, 1 + # gfx1250's WMMA catalog uses the 16x16x32 atom; the other swept targets + # support 16x16x16. + warp_tile_k = 32 if arch == "gfx1250" else 16 + # rocKE requires block_size == warp_m*warp_n*warp_k * wave_size. + block_size = warp_m * warp_n * warp_k * wave_size + spec = UniversalGemmSpec( + name="cod_smoke", + tile=TileSpec( + tile_m=128, + tile_n=128, + tile_k=32, + warp_m=warp_m, + warp_n=warp_n, + warp_k=warp_k, + warp_tile_m=16, + warp_tile_n=16, + warp_tile_k=warp_tile_k, + ), + trait=TraitSpec(pipeline=pipeline, epilogue="default"), + data=DataSpec(dtype_a="fp16", dtype_b="fp16", dtype_c="fp16", dtype_acc="fp32"), + wave_size=wave_size, + block_size=block_size, + batched=False, + ) + kernel = build_universal_gemm(spec, arch=arch) + return lower_kernel_to_llvm(kernel, arch=arch, llvm_flavor=flavor), kernel.name + + +def _patch(module: Any, name: str, value: Any) -> None: + """Replace an existing module attribute, refusing to invent a new one. + + Every override here redirects rocKE away from system ROCm. Were the symbol + renamed upstream, a plain assignment would still succeed and leave the probe + quietly measuring the wrong toolchain. + """ + if not hasattr(module, name): + raise AttributeError(f"{module.__name__}.{name} is gone; probe needs an update") + setattr(module, name, value) + + +def _pin_comgr_flavor_metadata(comgr: Any, flavor: str) -> None: + """Prevent rocKE's `/opt/rocm` fallback from misclassifying COD comgr. + + A COD root without ``.info/version`` makes rocKE consult unrelated system + metadata and reject matching IR before invoking comgr. Only for that case: + pinning the version also satisfies rocKE's own IR-flavor guard, so applying + it when the COD does carry trustworthy metadata would suppress the + comgr-vs-clang mismatch this suite exists to report. + """ + version = (7, 2) if flavor == "llvm22" else (7, 1) + _patch(comgr, "resolved_lib_rocm_version", lambda: version) + + +def _build_hsaco(arch: str, flavor: str) -> tuple[bytes, str, str] | None: + """(HSACO, kernel symbol, resolved flavor); None after emitting the red row.""" + try: + flavor = _comgr_flavor(flavor) + except Exception as exc: # noqa: BLE001 + _emit(arch, 1, f"flavor resolution failed: {exc}") + return None + try: + ir, symbol = _lower_ir(arch, flavor) + except Exception as exc: # noqa: BLE001 + _maybe_traceback() + _emit(arch, 1, f"lower failed: {exc}") + return None + try: + from rocke.runtime import comgr + + if os.environ.get("ROCKE_COMGR_VERSION_TRUSTED") == "0": + _pin_comgr_flavor_metadata(comgr, flavor) + hsaco, _timings = comgr.build_hsaco_from_llvm_ir( + ir, isa=f"amdgcn-amd-amdhsa--{arch}" + ) + except Exception as exc: # noqa: BLE001 + _maybe_traceback() + _emit(arch, 1, f"comgr compile failed ({flavor}): {exc}") + return None + if not hsaco: + _emit(arch, 1, "comgr produced empty HSACO") + return None + print(f"comgr lib: {comgr.resolved_lib_path()}", file=sys.stderr) + return hsaco, symbol, flavor + + +def _run_codegen(arch: str, flavor: str, clang: str, out_dir: str | None) -> None: + try: + ir, _ = _lower_ir(arch, flavor) + except Exception as exc: # noqa: BLE001 - report any lowering failure as a row + _maybe_traceback() + _emit(arch, 1, f"lower failed: {exc}") + return + + # Falling back to a bare shared /tmp would write predictable {arch}.ll/.o + # names another user on these lab machines could pre-plant as symlinks. + if out_dir: + out = Path(out_dir) + else: + # A caller-given --out is theirs to keep; one we invent is ours to remove. + out = Path(tempfile.mkdtemp(prefix="rocke-cod-")) + atexit.register(shutil.rmtree, out, True) + out.mkdir(parents=True, exist_ok=True) + ll_path = out / f"{arch}.ll" + obj_path = out / f"{arch}.o" + ll_path.write_text(ir, encoding="utf-8") + + # The COD clang drives the same AMDGPU backend as a standalone llc and is + # already hard-gated, so this needs no extra tool. `-x ir` marks .ll as IR. + cmd = [ + clang, "-x", "ir", str(ll_path), "-c", + "--target=amdgcn-amd-amdhsa", f"-mcpu={arch}", "-o", str(obj_path), + ] + print("+", " ".join(cmd), file=sys.stderr) + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300) + except (OSError, subprocess.SubprocessError) as exc: + _emit(arch, 1, f"clang did not run: {exc}") + return + if proc.stderr: + print(proc.stderr, file=sys.stderr) + + if proc.returncode != 0: + _emit(arch, 1, f"clang exit {proc.returncode}") + elif not obj_path.exists() or obj_path.stat().st_size == 0: + _emit(arch, 1, "clang produced no object") + else: + _emit(arch, 0, f"object {obj_path.stat().st_size} bytes") + + +def _run_comgr(arch: str, flavor: str) -> None: + built = _build_hsaco(arch, flavor) + if built is None: + return + hsaco, symbol, flavor = built + _emit(arch, 0, f"HSACO {len(hsaco)} bytes ({flavor})") + + # Optional on-device load, only when this host's device matches the target. + # Probing for a device is separate from loading onto one: a host with no + # ROCm agent is not a failure of this host-only lane. + try: + from rocke.runtime.hip_module import get_device_arch + + device_arch = get_device_arch(0) + except Exception: # noqa: BLE001 - no runtime, no device: nothing to load + return + if device_arch != arch: + return + try: + from rocke.runtime.hip_module import Runtime + + mod = Runtime().load_module(hsaco) + try: + mod.get_function(symbol) + finally: + mod.unload() + _emit(f"{arch}-device-load", 0, "loaded on device; kernel symbol found") + except Exception as exc: # noqa: BLE001 - a real load failure is a data row + _emit(f"{arch}-device-load", 1, f"device load failed: {exc}") + + +def _run_occupancy(arch: str, flavor: str, readelf: str) -> None: + """Compile with the COD comgr, then report the HSACO's codegen resources.""" + built = _build_hsaco(arch, flavor) + if built is None: + return + hsaco = built[0] + + try: + from rocke.benchmark.perf import occupancy + except Exception as exc: # noqa: BLE001 - report API drift as a data row + _emit(arch, 1, f"rocke.benchmark.perf.occupancy unavailable: {exc}") + return + + try: + # rocKE currently prefers /opt/rocm's readelf over PATH. Override its + # private resolver so this COD probe uses the binary whose provenance + # run_rocke.sh validated. Assigning a missing attribute would succeed + # silently and hand the probe back to system ROCm, so insist it is there. + _patch(occupancy, "_readelf", lambda: readelf) + res = occupancy.resources(hsaco, arch) + vspill = int(res.get("vgpr_spill") or 0) if res else 0 + sspill = int(res.get("sgpr_spill") or 0) if res else 0 + except Exception as exc: # noqa: BLE001 - probe failure is a data row + _maybe_traceback() + _emit(arch, 1, f"occupancy probe failed: {exc}") + return + if not res: + _emit(arch, 1, "ELF notes unreadable (need a working llvm-readelf)") + return + summary = ( + f"vgpr={res.get('vgpr')} agpr={res.get('agpr')} sgpr={res.get('sgpr')} " + f"lds={res.get('lds_bytes')}B spill={vspill}/{sspill} occ={res.get('occupancy')}" + ) + if vspill or sspill: + _emit(arch, 1, f"register spill on fixed smoke kernel: {summary}") + else: + _emit(arch, 0, summary) + + +def main() -> int: + ap = argparse.ArgumentParser(description="rocKE<->COD interop smoke (single arch)") + ap.add_argument("--mode", required=True, choices=("codegen", "comgr", "occupancy")) + ap.add_argument("--arch", required=True) + ap.add_argument("--flavor", default="llvm22") + ap.add_argument("--clang", default="clang", help="COD clang (codegen mode)") + ap.add_argument( + "--readelf", default="llvm-readelf", help="COD llvm-readelf (occupancy mode)" + ) + ap.add_argument("--out", help="object dir (codegen); default: a private temp dir") + ap.add_argument( + "--experimental", + action="store_true", + help="tag result rows as experimental (non-production arch)", + ) + args = ap.parse_args() + + if args.experimental: + global _SUBTEST_SUFFIX + _SUBTEST_SUFFIX = " (experimental)" + + # Encode the compile path in the group so codegen and comgr rows stay distinct + # in a consolidated suite. The extractor keys its by-area breakdown on the + # first dotted component, so the area stays "universal_gemm". + global GROUP + if args.mode == "codegen": + GROUP = f"{GROUP}.codegen" + _run_codegen(args.arch, args.flavor, args.clang, args.out) + elif args.mode == "occupancy": + GROUP = _OCCUPANCY_GROUP + _run_occupancy(args.arch, args.flavor, args.readelf) + else: + GROUP = f"{GROUP}.comgr" + _run_comgr(args.arch, args.flavor) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/rocke/rocke_junit_results.py b/bin/rocke/rocke_junit_results.py new file mode 100755 index 000000000..5aa5105b3 --- /dev/null +++ b/bin/rocke/rocke_junit_results.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Convert a JUnit XML report (pytest --junitxml or ctest --output-junit) into the +# canonical rocKE CI result lines consumed by extract-rocke.sh: +# +# ROCKE_RESULT||||| +# +# status is 0 (pass) or 1 (fail). A skip is a pass ("skipped: ...") only when the +# *host* is what is missing -- no GPU of the right arch, no ROCm torch -- so a +# GPU-free host stays green. A skip whose reason names the toolchain is a red +# "blocked: ..." row instead: it means the compiler under test could not do the +# work, which is the one thing this CI exists to catch. --relevance attaches the +# per-test evidence rocke_relevance.py recorded; --relevance-default covers lanes +# with none. + +from __future__ import annotations + +import argparse +import json +import re +import xml.etree.ElementTree as ET + +from rocke_relevance import ( + MANIFEST_VERSION, + TIER_COMPILER, + TIER_HARNESS, + TIER_ORDER, + TIER_UNMEASURED, +) +from rocke_result import emit as _emit + +# Toolchain markers in a skip reason. Kept deliberately narrow and extended only +# from reasons actually observed, so an unrecognised reason stays green rather than +# turning the nightly red on a guess. +_BLOCKED_SKIP = re.compile( + r"comgr|hipcc|\bclang\b|llvm|\bisa\b|cannot target|compile unavailable" + r"|rocke_engine|c\+\+ engine|byte.identity|datalayout", + re.IGNORECASE, +) + +# Of those, the ones naming an artifact *this CI* is responsible for building. Still +# red -- the tests did not run -- but ours to fix, so pointing triage at the compiler +# would waste its time. A COD failure while building it is reported by the builder. +_BLOCKED_OURS = re.compile(r"rocke_engine|c\+\+ engine", re.IGNORECASE) + + +def _at_least(tier: str, floor: str) -> str: + """The more compiler-relevant of the two, when a floor is given. + + Some lanes drive the toolchain by construction -- every case in the numeric lane + compiles a kernel and launches it -- yet the measured evidence can be empty + because the work happened in a child process. Reporting such a row as `logic` + ("not the compiler's business") is worse than reporting no measurement at all, + so the lane's guarantee wins. + """ + if not floor or floor not in TIER_ORDER or tier not in TIER_ORDER: + return tier + return min(tier, floor, key=TIER_ORDER.index) + + +def _load_manifest(path: str) -> tuple[dict[str, str], list[str]]: + """(key -> tier, setup problems worth a red row). + + An entry with no usable tier is dropped rather than kept as "", so it counts + as unjoined below instead of silently degrading the signal. + """ + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError) as exc: + return {}, [f"cannot read relevance manifest {path}: {exc}"] + # The plugin stamps the layout it wrote; refuse to read tiers out of a shape + # this decoder does not know rather than mislabel every row from it. + found = data.get("version") + if found != MANIFEST_VERSION: + return {}, [ + f"relevance manifest {path} is version {found!r}, expected {MANIFEST_VERSION}" + ] + tests = (data.get("tests") or {}).items() + tiers = {k: t for k, v in tests if (t := str(v.get("tier") or ""))} + return tiers, list(data.get("install_errors") or []) + + +def main() -> int: + ap = argparse.ArgumentParser(description="JUnit XML -> rocKE CI result lines") + ap.add_argument("--junit", required=True, help="path to the JUnit XML report") + ap.add_argument( + "--group-default", + default="tests", + help="group used when a testcase has no classname", + ) + ap.add_argument("--relevance", help="path to the rocke_relevance.py manifest") + ap.add_argument( + "--relevance-default", + default=TIER_UNMEASURED, + help="relevance for cases the manifest does not cover", + ) + ap.add_argument( + "--relevance-floor", + default="", + help="least relevance a case that ran may report, for a lane whose every " + "test drives the toolchain by construction (e.g. on-device numerics)", + ) + args = ap.parse_args() + + tiers: dict[str, str] = {} + problems: list[str] = [] + if args.relevance: + tiers, problems = _load_manifest(args.relevance) + for problem in problems: + _emit("setup", "relevance-probe", 1, problem, TIER_HARNESS) + + try: + root = ET.parse(args.junit).getroot() + except (OSError, ET.ParseError) as exc: + _emit( + "setup", + f"{args.group_default}-junit-parse", + 1, + f"cannot parse {args.junit}: {exc}", + TIER_HARNESS, + ) + return 0 + + seen = 0 + unjoined = 0 + for case in root.iter("testcase"): + seen += 1 + group = case.get("classname") or args.group_default + subtest = case.get("name") or "unnamed" + tier = tiers.get(f"{case.get('classname') or ''}\t{case.get('name') or ''}") + if tier is None: + tier = args.relevance_default + unjoined += 1 + tier = _at_least(tier, args.relevance_floor) + failure = case.find("failure") + error = case.find("error") + skipped = case.find("skipped") + status_attr = (case.get("status") or "").lower() + + if ( + failure is not None + or error is not None + or status_attr in ("fail", "failed") + ): + node = failure if failure is not None else error + msg = (node.get("message") if node is not None else "") or "failed" + _emit(group, subtest, 1, msg, tier) + elif skipped is not None or status_attr in ("notrun", "disabled", "skipped"): + reason = ( + skipped.get("message") if skipped is not None else "" + ) or "skipped" + if _BLOCKED_SKIP.search(reason): + blocked_tier = ( + TIER_HARNESS if _BLOCKED_OURS.search(reason) else TIER_COMPILER + ) + _emit(group, subtest, 1, f"blocked: {reason}", blocked_tier) + else: + # A test that never ran recorded no evidence, so the manifest tier + # is spurious -- and lands on 'logic', which reads as "not the + # compiler's business". Say outright that nothing was measured. + _emit(group, subtest, 0, f"skipped: {reason}", TIER_UNMEASURED) + else: + _emit(group, subtest, 0, "", tier) + + if seen == 0: + _emit( + "setup", + f"{args.group_default}-no-testcases", + 1, + "report contained no testcases", + TIER_HARNESS, + ) + # A manifest that stops joining silently loses the compiler signal, so say so + # instead of reporting everything as unmeasured. + elif tiers and unjoined: + _emit( + "setup", + "relevance-join", + 1, + f"{unjoined} of {seen} testcases had no relevance entry", + TIER_HARNESS, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/rocke/rocke_relevance.py b/bin/rocke/rocke_relevance.py new file mode 100644 index 000000000..b5016f516 --- /dev/null +++ b/bin/rocke/rocke_relevance.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Per-test COD-toolchain relevance for the rocKE nightly, as a pytest plugin. +# +# Whether a red row can be a compiler regression is a property of what a test +# *did*, not of what it imports: rocke/__init__.py eagerly imports the comgr, HIP +# and hipcc entry points, so import-graph analysis marks nearly every test capable +# and decides nothing (see README.md "Test relevance"). Measure it instead, in +# decreasing precision: +# 1. rocke.runtime._ctypes_bind._LazyFn.__call__ -- the chokepoint every comgr +# and HIP native call goes through; level-triggered, so it sees every call. +# 2. ctypes.dlopen / subprocess.Popen audit hooks -- a COD library load or a +# hipcc/clang/llvm-* spawn from anywhere, including code this plugin knows +# nothing about. Edge-triggered backstop. +# 3. the rocke_engine import -- the C++ engine is built by the COD. +# +# No allowlist, so a new test that compiles is flagged the moment it does. A +# chokepoint lost to a rocke refactor becomes an install error and then a red +# setup row, so staleness is loud rather than silent. + +from __future__ import annotations + +import json +import os +import re +import sys +from collections import defaultdict + +# Relevance of a result row, i.e. how a red row should be triaged. +TIER_COMPILER = "compiler" # this test drove the COD toolchain +TIER_CAPABLE = "compiler-capable" # its module did, this test did not get there +TIER_LOGIC = "logic" # no toolchain interaction anywhere in the module +TIER_HARNESS = "harness" # this CI's own plumbing +TIER_UNMEASURED = "unmeasured" # lane carries no per-test evidence + +# Most to least compiler-relevant, so a consumer can pick the stronger of two +# verdicts (see rocke_junit_results.py's floor) and order a report by triage value. +TIER_ORDER = (TIER_COMPILER, TIER_CAPABLE, TIER_UNMEASURED, TIER_HARNESS, TIER_LOGIC) + +MANIFEST_VERSION = 1 + +# Shared-library roles worth attributing; the path is kept too, since rocke may +# prefer a torch-bundled libamd_comgr over the COD one and the report must say so. +_LIB_ROLES = ( + ("comgr", re.compile(r"libamd_comgr|amd_comgr\.dll")), + ("hiprtc", re.compile(r"hiprtc")), + ("hip", re.compile(r"libamdhip64|amdhip64\.dll")), + ("hsa", re.compile(r"libhsa-runtime")), +) + +# Toolchain binaries; matched on argv0's basename. +_TOOL_BIN = re.compile( + r"^(hipcc|hipconfig|amdclang\+*|clang\+*|clang-\d+|opt|llc|lld|ld\.lld" + r"|llvm-[a-z0-9-]+|roc-obj[a-z-]*|rocprof[a-z0-9]*|amdgpu-arch|offload-arch)$" +) + +_SESSION = "" + +# Python interpreters, matched on argv0's basename: a test that hands its real work +# to a child process (rocKE's numeric and gfx950 smoke tests both do) does all its +# compiling there, where the in-process chokepoint cannot see any of it. +_PY_BIN = re.compile(r"^python[0-9.]*(\.exe)?$") + + +def _rocke_tree() -> str: + """Root of the rocKE checkout, or "" when it cannot be located.""" + if _S.tree is None: + _S.tree = "" + try: + import rocke + + # /platform/python/rocke/__init__.py -> + _S.tree = os.path.realpath( + os.path.join(os.path.dirname(rocke.__file__), "..", "..", "..") + ) + except Exception: # noqa: BLE001 - never break a spawn + pass + return _S.tree + + +def _rocke_child(argv) -> str: # noqa: ANN001 + """What a spawned interpreter will run, when that code lives in the rocKE tree. + + Resolved against sys.path rather than matched by name, so a renamed package + keeps being recognised -- exactly the staleness this design is meant to avoid. + """ + tree = _rocke_tree() + if not tree: + return "" + roots = [p for p in sys.path if p and os.path.realpath(p).startswith(tree)] + for i, raw in enumerate(argv): + arg = os.fsdecode(raw) + if arg == "-m" and i + 1 < len(argv): + target = os.fsdecode(argv[i + 1]) + top = target.split(".", 1)[0] + if any( + os.path.isdir(os.path.join(r, top)) + or os.path.exists(os.path.join(r, f"{top}.py")) + for r in roots + ): + return target + elif arg.endswith(".py") and os.path.realpath(arg).startswith(tree): + return os.path.basename(arg) + return "" + + +class _State: + def __init__(self) -> None: + self.current: list[str] = [] + self.evidence: dict[str, set[str]] = defaultdict(set) + self.module_of: dict[str, str] = {} + self.install_errors: list[str] = [] + self.tree: str | None = None + + +_S = _State() + + +def _record(kind: str) -> None: + _S.evidence[_S.current[-1] if _S.current else _SESSION].add(kind) + + +def _audit(event, args): # noqa: ANN001 + if event == "subprocess.Popen": + exe = os.fsdecode(args[0]) if args[0] else "" + argv = args[1] or [] + base = os.path.basename(exe or (os.fsdecode(argv[0]) if argv else "")) + if _TOOL_BIN.match(base): + _record(f"spawn:{base}") + elif _PY_BIN.match(base): + target = _rocke_child(argv) + if target: + _record(f"spawn:py:{target}") + elif event == "ctypes.dlopen": + name = str(args[0]) if args else "" + for role, pat in _LIB_ROLES: + if pat.search(name): + _record(f"dlopen:{role}") + break + elif event == "import" and args and args[0] == "rocke_engine": + _record("engine") + + +def _install_native_probe() -> None: + """Wrap the comgr/HIP call chokepoint (channel 1).""" + from rocke.runtime import _ctypes_bind + + cls = _ctypes_bind._LazyFn + original = cls.__call__ + + def probed(self, *args, **kwargs): + resolver = getattr(self._lib_resolver, "__module__", "") or "" + family = "comgr" if resolver.endswith("comgr") else ( + "hip" if resolver.endswith("hip_module") else "native" + ) + _record(f"call:{family}") + return original(self, *args, **kwargs) + + cls.__call__ = probed + + +def _mangle(nodeid: str) -> tuple[str, str]: + """(classname, name) exactly as pytest's junitxml writes them, so the + manifest joins onto the JUnit report. Mirrors _pytest.junitxml.""" + path, bracket, params = nodeid.partition("[") + names = path.split("::") + names[0] = names[0].replace(os.sep, ".").replace("/", ".") + names[0] = re.sub(r"\.py$", "", names[0]) + names[-1] += bracket + params + return ".".join(names[:-1]), names[-1] + + +# --- pytest hooks ---------------------------------------------------------- + + +def pytest_configure(config): # noqa: ANN001, ARG001 + sys.addaudithook(_audit) + try: + _install_native_probe() + except Exception as exc: # noqa: BLE001 + _S.install_errors.append(f"comgr/HIP call probe not installed: {exc!r}") + + +def pytest_collection_modifyitems(session, config, items): # noqa: ANN001, ARG001 + for item in items: + try: + _S.module_of[item.nodeid] = str(item.path) + except Exception: # noqa: BLE001 + _S.module_of[item.nodeid] = "" + + +def pytest_runtest_logstart(nodeid, location): # noqa: ANN001, ARG001 + _S.current.append(nodeid) + + +def pytest_runtest_logfinish(nodeid, location): # noqa: ANN001, ARG001 + if _S.current: + _S.current.pop() + + +def pytest_sessionfinish(session, exitstatus): # noqa: ANN001, ARG001 + out = os.environ.get("ROCKE_RELEVANCE_OUT") + if not out: + return + + # A test failing before its compile step leaves no evidence of its own, but + # its module's other tests still show whether the area drives the COD -- so + # promote it to compiler-capable rather than writing it off as pure logic. + touched_modules = { + _S.module_of.get(nid, "") + for nid, kinds in _S.evidence.items() + if kinds and nid in _S.module_of + } + touched_modules.discard("") + + tests: dict[str, dict[str, object]] = {} + for nid, module in _S.module_of.items(): + kinds = sorted(_S.evidence.get(nid, ())) + if kinds: + tier = TIER_COMPILER + elif module in touched_modules: + tier = TIER_CAPABLE + else: + tier = TIER_LOGIC + classname, name = _mangle(nid) + tests[f"{classname}\t{name}"] = {"tier": tier, "evidence": kinds} + + payload = { + "version": MANIFEST_VERSION, + "tests": tests, + "install_errors": _S.install_errors, + } + tmp = f"{out}.tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(payload, fh) + os.replace(tmp, out) diff --git a/bin/rocke/rocke_result.py b/bin/rocke/rocke_result.py new file mode 100644 index 000000000..c6d69d174 --- /dev/null +++ b/bin/rocke/rocke_result.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# The canonical rocKE CI result line consumed by extract-rocke.sh: +# +# ROCKE_RESULT||||| +# +# A newline in the message is encoded as US (0x1F) and a literal '|' as RS (0x1E), +# so a multi-line message survives as one line; extract-rocke.sh restores both. +# +# says whether a red row can be a compiler-of-the-day regression at +# all; see rocke_relevance.py for the vocabulary. Empty when a lane has none. + +from __future__ import annotations + +import os +import sys + +# In-message sentinels (see module header). rocke_extract.py decodes with these. +NL = "\x1f" +PIPE = "\x1e" + + +def sanitize(text: str) -> str: + # rstrip real whitespace *before* encoding '|' (0x1E is whitespace to + # str.rstrip, so encoding first would silently drop a trailing pipe). + lines = [ + line.rstrip() + for line in str(text).replace("\r\n", "\n").replace("\r", "\n").split("\n") + ] + while lines and not lines[0]: + lines.pop(0) + while lines and not lines[-1]: + lines.pop() + return NL.join(lines).replace("|", PIPE) + + +def _record(line: str) -> None: + """Copy a row to the worker's row log, when it asked for one. + + Lets run_rocke.sh tally its own run for the human summary without capturing + stdout. Best-effort by design: a tally is a convenience and must never turn + a good result into a failure. + """ + path = os.environ.get("ROCKE_ROW_LOG") + if not path: + return + try: + with open(path, "a", encoding="utf-8") as fh: + fh.write(line + "\n") + except OSError: + pass + + +def emit( + group: str, subtest: str, status: int, message: str = "", relevance: str = "" +) -> None: + line = ( + f"ROCKE_RESULT|{sanitize(group)}|{sanitize(subtest)}|{status}" + f"|{sanitize(message)}|{sanitize(relevance)}" + ) + print(line) + sys.stdout.flush() + _record(line) diff --git a/bin/run_rocke.sh b/bin/run_rocke.sh new file mode 100755 index 000000000..104c7b706 --- /dev/null +++ b/bin/run_rocke.sh @@ -0,0 +1,1183 @@ +#!/usr/bin/env bash +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# rocKE driver: run one lane against the compiler-of-the-day (COD), emit +# canonical "ROCKE_RESULT|group|subtest|status|message|relevance" lines and +# close with a short summary. Self-contained, so an engineer can run it by hand; +# the nightly drives the same lanes. The worker modules it runs are in bin/rocke; +# the ROCKE_* knobs default just below. The +# CI-side extractor that turns the rows into dashboard rows, and the full +# documentation, live in the apps repo under openmp-ci/rocKE (extract-rocke.sh, +# README.md). +# +# Usage: run_rocke.sh [-r] [-u] ; -h lists the lanes and the common knobs. +# 'all' runs every lane in one process for a single consolidated report/mail +# (see USER-run-rocKE-all); schedule the per-lane wrappers for one report each. +# The -r/-u flags mirror CK's; the nightly wrappers set both by env instead. + +set -u + +# Single source of truth for the stage check and the help text. +Lanes="all|engine|ctest|pytest|gpu-numeric|perf|cod-codegen|cod-comgr" + +function printUsage { + cat < + +Runs one rocKE lane against the compiler-of-the-day and prints ROCKE_RESULT rows; +'all' runs every lane and adds a pass/total tally per lane. A failing test is a +result row, not a driver error, so this exits 0 either way: read the closing +summary or the rows, not \$?. + + -r rebuild each lane from a clean build dir (ROCKE_REBUILD=1; off by default) + -u refresh the shared rocm-libraries checkout (ROCKE_UPDATE_REPO=1; off by default) + -h this help + +Common knobs (every ROCKE_* default is set together at the top of this script; +the lane table and the full list are in openmp-ci/rocKE/README.md): + AOMP= COD compiler under test + ROCKE_ALL_LANES='...' lanes 'all' runs, in order + ROCKE_CI_ARCHES='...' arch sweep for the cod-*/perf lanes + ROCKE_TOP= rocKE platform checkout to test (else one is cloned) + ROCKE_CI_BUILD_ROOT= out-of-tree build root + ROCKE_VENV= the only interpreter this script may install into + ROCKE_DEBUG=1 full Python tracebacks from the cod-*/perf lanes +EOF +} + +# Leading -r/-u are sugar for the ROCKE_REBUILD / ROCKE_UPDATE_REPO gates the +# worker reads below; the 'all' fork exports them, so lane children inherit the +# same choice. +while [[ "${1:-}" == -?* ]]; do + case "$1" in + -r) export ROCKE_REBUILD=1 ;; + -u) export ROCKE_UPDATE_REPO=1 ;; + -h|--help) printUsage; exit 0 ;; + --) shift; break ;; + *) echo "unknown option: $1"; printUsage; exit 2 ;; + esac + shift +done + +Stage="${1:-}" +ScriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# The worker modules keep their rocke_ prefix: they go on the PYTHONPATH of +# rocKE's own test session, where a plain result.py would shadow the project's. +HelperDir="${ScriptDir}/rocke" +InheritedPythonPath="${PYTHONPATH:-}" +if [[ -z "${Stage}" ]]; then + printUsage; exit 2 +elif [[ "|${Lanes}|" != *"|${Stage}|"* ]]; then + echo "unknown stage: ${Stage}"; printUsage; exit 2 +fi + +# Compiler-of-the-day (COD) toolchain. A single knob, AOMP, selects the compiler +# under test; every other tool (comgr, HIP runtime, hipcc, device-libs) is derived +# from the install that ships it, so no lane can silently fall back to a system +# ROCm (/opt/rocm) or /usr toolchain. The assertCodToolchain gate below proves it. +: "${AOMP:=/COD/LATEST/aomp/llvm}" +AompInput="${AOMP}" +AOMP="$(realpath -m "${AompInput}")" +export AOMP + +# Walk up from the resolved llvm dir to the nearest ancestor shipping comgr or +# hipcc. Key on those, not on include/hip or amdgcn/bitcode: some packagings put +# headers and device-libs under llvm/, so keying on those would stop one level +# below the real root and let comgr fall back to a system /opt/rocm. +function resolveRocmRoot { + local Dir="${1}" Start="${1}" _Hop + for _Hop in 0 1 2 3; do + if [[ -e "${Dir}/lib/libamd_comgr.so" || -x "${Dir}/bin/hipcc" ]]; then + echo "${Dir}"; return 0 + fi + Dir="$(realpath -m "${Dir}/..")" + done + realpath -m "$(dirname "${Start}")" # give up; prefix check + hygiene flag it +} + +# ROCM_PATH (the house-standard knob) may override the derived root, but only +# while realpath(ROCM_PATH) is a prefix of realpath(AOMP): otherwise a stray +# ambient `export ROCM_PATH=/opt/rocm` would hijack the root and wave a system +# ROCm through the hygiene gate as [COD]. +DerivedRoot="$(resolveRocmRoot "${AOMP}")" +if [[ -n "${ROCM_PATH:-}" ]]; then + EnvRoot="$(realpath -m "${ROCM_PATH}")" + if [[ "${AOMP}/" == "${EnvRoot}/"* ]]; then + RocmRoot="${EnvRoot}"; RocmRootSource="from env ROCM_PATH" + else + echo "WARNING: ignoring ROCM_PATH=${EnvRoot} -- not a prefix of AOMP=${AOMP}" + echo " (looks like an ambient/system ROCm). Using the AOMP-derived root." + echo " To force a specific root, point ROCM_PATH at an ancestor of AOMP." + RocmRoot="${DerivedRoot}"; RocmRootSource="derived from AOMP (ROCM_PATH ignored)" + fi +else + RocmRoot="${DerivedRoot}"; RocmRootSource="derived from AOMP" +fi + +export ROCM_PATH="${RocmRoot}" +export HIP_PATH="${RocmRoot}" +# Pin hipcc's clang to the COD llvm so the HIP path can never pick a system clang. +export HIP_CLANG_PATH="${AOMP}/bin" +export ROCKE_COMGR_LIB="${ROCKE_COMGR_LIB:-${RocmRoot}/lib/libamd_comgr.so}" +export ROCKE_HIP_LIB="${ROCKE_HIP_LIB:-${RocmRoot}/lib/libamdhip64.so}" +export CC="${AOMP}/bin/clang" +export CXX="${AOMP}/bin/clang++" +# COD llvm tools first, then the install bin (hipcc, rocprofv3), then the rest. +export PATH="${AOMP}/bin:${RocmRoot}/bin:${PATH}" +# AOMP compiler runtime first (libomp/libomptarget), then the pinned ROCm +# runtime (libamdhip64/libhsa). Guard the tail: an unset var must not leave a +# trailing ':' -- an empty entry means CWD, which would breach COD isolation. +export LD_LIBRARY_PATH="${AOMP}/lib:${RocmRoot}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + +# rocKE's CMakeLists uses block(), which needs CMake >= 3.25; prefer a modern +# local cmake when the distro one is older. +: "${ROCKE_CMAKE_BIN:=${HOME}/local/cmake/bin}" +[[ -x "${ROCKE_CMAKE_BIN}/cmake" ]] && export PATH="${ROCKE_CMAKE_BIN}:${PATH}" + +: "${AOMP_REPOS_TEST:=${HOME}/git/aomp-test}" +: "${ROCKE_TOP:=${AOMP_REPOS_TEST}/composable-kernels/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/platform}" +# -s keeps this in ROCKE_TOP's path namespace. Resolving symlinks can put the +# library test root under a different prefix (/work/... vs /home/...), which makes +# pytest root its collection tree at / and scan shared parents like /work, +# aborting on the first unreadable entry there. +ROCKE_PROJECT_ROOT="$(realpath -m -s "${ROCKE_TOP}/..")" +# Path from the shared rocm-libraries repo root down to the rocKE platform dir. +ROCKE_TOP_SUFFIX="/dnn-providers/hip-kernel-provider/rocke/platform" +ROCKE_REPO_ROOT="${ROCKE_TOP%"${ROCKE_TOP_SUFFIX}"}" +: "${ROCKE_VENV:=${HOME}/.local/rocKE-venv}" +: "${ROCKE_CODEGEN_FLAVOR:=auto}" +: "${ROCKE_COMGR_FLAVOR:=auto}" +: "${ROCKE_CI_ARCHES:=gfx950 gfx942 gfx1151 gfx1201}" +# Experimental arches, appended to the COD compile lanes only (not part of the +# production sweep); each compiles through the COD and exercises the on-device +# HSACO load when the runner matches. Set empty to disable. +: "${ROCKE_CI_ARCHES_EXPERIMENTAL=gfx90a gfx1250}" +: "${ROCKE_ENGINE_FLAVORS:=llvm20 llvm22}" +# Sibling of the rocm-libraries checkout, matching the ck-src/ck-build layout +# beside it. Off /tmp, so no reaper can drop the tree or its root marker. +if [[ "${ROCKE_REPO_ROOT}" == "${ROCKE_TOP}" && -z "${ROCKE_CI_BUILD_ROOT:-}" ]]; then + # A ROCKE_TOP outside a rocm-libraries checkout leaves nothing to sit beside, + # so the derived root would land inside the engineer's own tree. + echo "WARNING: ROCKE_TOP is not inside a rocm-libraries checkout; the build root" + echo " will be derived next to it. Set ROCKE_CI_BUILD_ROOT to choose one." +fi +: "${ROCKE_CI_BUILD_ROOT:=${ROCKE_REPO_ROOT%/*}/rocke-build}" +ROCKE_CI_BUILD_ROOT="$(realpath -m "${ROCKE_CI_BUILD_ROOT}")" +# Lanes the 'all' meta-stage runs, in order: host-only lanes first, then the +# on-device numeric lane, with the host-only perf footprint last. Override to +# scope a run (e.g. ROCKE_ALL_LANES='engine pytest'). +: "${ROCKE_ALL_LANES:=engine ctest pytest cod-codegen cod-comgr gpu-numeric perf}" + +# Run-shape knobs. The nightly wrappers pin both gates to 1; run by hand they +# default off, so a rerun neither wipes the lane build dir nor touches the shared +# checkout, and -r/-u opt in exactly as they do for CK's driver. +: "${ROCKE_REBUILD:=0}" +: "${ROCKE_UPDATE_REPO:=0}" +: "${ROCKE_SETUP_VENV:=1}" +: "${ROCKE_REPO_URL:=https://github.com/ROCm/rocm-libraries.git}" +: "${ROCKE_REPO_BRANCH:=develop}" +: "${ROCKE_TORCH_INDEX_URL:=}" +# rocKE uses these without declaring them in its dev extras. +: "${ROCKE_EXTRA_TEST_DEPS:=pyarrow}" +# A concurrent run holds the source lock for its whole duration -- deliberately, +# so the tree cannot change under a test in flight -- so allow a nightly's worth +# of waiting, then fail with a red row rather than hang. +: "${ROCKE_LOCK_WAIT:=3600}" + +# The interop lanes want deterministic reference IR: the C++ engine is +# byte-identical but not built here, so use the Python backend and skip the +# noisy fallback warning. +export ROCKE_BACKEND="${ROCKE_BACKEND:-python}" +export PYTHONPATH="${ROCKE_TOP}/python:${ROCKE_PROJECT_ROOT}/library" +export PYTHONPATH+="${InheritedPythonPath:+:${InheritedPythonPath}}" + +BuildRoot="$(realpath -m "${ROCKE_CI_BUILD_ROOT}/${Stage}")" +PyBin="" + +# Every row emitted anywhere in the run is copied here (by this function, and by +# rocke_result.py via ROCKE_ROW_LOG) so the closing summary can tally the run +# without capturing its own stdout. Empty until a lane opens one. +RowLog="" + +# Canonical result line consumed by extract-rocke.sh; '|' is the field +# separator, so strip it from caller-supplied fields. Relevance says how a red +# row should be triaged (see rocke_relevance.py); it defaults to this CI's own +# plumbing because that is what the bash-side rows are. +function rockeResult { + local Group="${1//|/ }" Subtest="${2//|/ }" Status="${3}" Message="${4//|/ }" + local Relevance="${5:-harness}" Row + Row="ROCKE_RESULT|${Group}|${Subtest}|${Status}|${Message}|${Relevance//|/ }" + echo "${Row}" + if [[ -n "${RowLog}" ]]; then printf '%s\n' "${Row}" >> "${RowLog}"; fi +} + +# A setup failure is a red data row, not a harness crash, so still exit 0. +function fatalSetup { + echo "Error: ${1}" + rockeResult setup "${2}" 1 "${1}" + exit 0 +} + +# Directory locks avoid inheritable file descriptors, so an external tool (or a +# daemon it starts) cannot retain a nightly lock after this worker exits. +HeldLockDirs=() +# shellcheck disable=SC2317 # invoked indirectly by the EXIT trap +function cleanupOnExit { + local Lock Owner + for Lock in "${HeldLockDirs[@]}"; do + Owner="${Lock}/owner" + if [[ -f "${Owner}" && "$(< "${Owner}")" == "${BASHPID}" ]]; then + rm -f "${Owner}" + rmdir "${Lock}" 2>/dev/null || true + fi + done + [[ -n "${RowLog}" ]] && rm -f "${RowLog}" + return 0 +} +trap cleanupOnExit EXIT + +function acquireDirLock { # + local Lock="${1}" Description="${2}" OwnerPid Entry Stale Waited=0 Announced=0 + while ! mkdir "${Lock}" 2>/dev/null; do + [[ ! -L "${Lock}" ]] \ + || fatalSetup "refusing symlinked ${Description} lock: ${Lock}" lock + (( Waited < ROCKE_LOCK_WAIT )) || fatalSetup \ + "gave up after ${ROCKE_LOCK_WAIT}s waiting for the ${Description} lock: ${Lock}" \ + lock + OwnerPid="$(cat "${Lock}/owner" 2>/dev/null || true)" + # /proc, not kill -0: kill reports EPERM for a live process owned by another + # user, and treating that as dead would quarantine a held lock and let two + # runs into the same tree. + if [[ "${OwnerPid}" =~ ^[0-9]+$ && -d "/proc/${OwnerPid}" ]]; then + # Say it once: an unexplained silent wait looks like a hang. + (( Announced )) || echo "waiting for the ${Description} lock held by pid ${OwnerPid}: ${Lock}" + Announced=1 + sleep 1; (( ++Waited )) + continue + fi + if [[ ! "${OwnerPid}" =~ ^[0-9]+$ ]]; then + sleep 1; (( ++Waited )) + [[ ! -e "${Lock}" ]] && continue + OwnerPid="$(cat "${Lock}/owner" 2>/dev/null || true)" + [[ "${OwnerPid}" =~ ^[0-9]+$ ]] || fatalSetup \ + "malformed ${Description} lock owner: ${Lock}" lock + continue + fi + Entry="$(find "${Lock}" -mindepth 1 -maxdepth 1 ! -name owner -print -quit 2>/dev/null)" + if [[ -d "${Lock}" && -z "${Entry}" ]]; then + Stale="${Lock}.stale.${BASHPID}" + [[ ! -e "${Stale}" ]] \ + || fatalSetup "stale-lock quarantine already exists: ${Stale}" lock + if mv "${Lock}" "${Stale}" 2>/dev/null; then + rm -f "${Stale}/owner" + rmdir "${Stale}" 2>/dev/null \ + || fatalSetup "cannot remove stale ${Description} lock: ${Stale}" lock + continue + fi + # Someone else got there first, or the directory is not ours to move: + # retry on the same terms as a live owner rather than spinning on the CPU. + sleep 1; (( ++Waited )) + continue + fi + fatalSetup "cannot recover stale ${Description} lock: ${Lock}" lock + done + printf '%s\n' "${BASHPID}" > "${Lock}/owner" \ + || fatalSetup "cannot record ${Description} lock owner: ${Lock}" lock + HeldLockDirs+=("${Lock}") +} + +# A directory that exists is not necessarily one pytest can build a collector +# for; an argument it cannot collect from aborts the whole run. Require visible +# test files, matching pytest's default python_files patterns. +function hasTests { # + [[ -d "${1}" && -r "${1}" ]] || return 1 + [[ -n "$(find "${1}" \( -name 'test_*.py' -o -name '*_test.py' \) \ + -print -quit 2>/dev/null)" ]] +} + +# pytest reports a usage error (an unopenable root, conftest or plugin) on its +# output only, so a row saying just "exited with status 4" is undiagnosable. +function runnerDetail { # + local Log="${1:-}" Line + [[ -n "${Log}" && -f "${Log}" ]] || return 0 + Line="$(grep -m1 -E '^(ERROR|ImportError while loading)' "${Log}")" || return 0 + [[ -n "${Line}" ]] && printf ': %s' "${Line:0:160}" +} + +# Run pytest from the tests dir under the relevance plugin, teeing to so a +# usage error stays diagnosable. Returns pytest's own status. +# Build rocKE's C++ engine extension (`rocke_engine`) and echo the directory +# holding it, so the pytest lanes can import it. +# +# rocKE's cross-engine tests skip without it, and it is not a side concern: the +# extension is 200k lines of C++ compiled by the COD, and the tests it unlocks +# compare the COD-built engine against the Python one. Built through rocKE's own +# ROCKE_BUILD_PYBIND option -- one tree yields both the archive and the module -- so +# there is no second recipe of ours to keep in step with theirs. +# +# Shared across lanes and reused while it is newer than rocKE's C++ sources, so the +# 'all' run pays for it once and a hand rerun pays nothing. +# +# Reports the directory in EngineExtDir rather than on stdout: this function also +# prints progress and, on failure, a result row, and a caller capturing stdout would +# swallow the row into a variable instead of the log. +EngineExtDir="" +function ensureEngineExtension { # sets EngineExtDir + local Root="${ROCKE_CI_BUILD_ROOT}/engine-ext" Ext + EngineExtDir="" + Ext="$(find "${Root}" -name 'rocke_engine*.so' -print -quit 2>/dev/null)" + # Ask directly whether any C++ source is newer than the module we already have, + # rather than sorting the tree: one stat walk, and no filename can confuse it. + if [[ -n "${Ext}" ]] && [[ -z "$(find "${ROCKE_TOP}/cpp" -newer "${Ext}" \ + \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' -o -name 'CMakeLists.txt' \) \ + -print -quit 2>/dev/null)" ]]; then + EngineExtDir="$(dirname "${Ext}")"; return 0 + fi + # Both the lock and the tree live under the (validated) build root; create it + # first so the lock's own mkdir cannot fail for a missing parent. + mkdir -p "${Root}" + acquireDirLock "${Root}.lock" "engine extension build" + local PyBind + PyBind="$("${PyBin}" -m pybind11 --cmakedir 2>/dev/null)" + if [[ -z "${PyBind}" ]]; then + rockeResult setup engine-extension 1 \ + "pybind11 unavailable in ${ROCKE_VENV}; rocKE's cross-engine tests cannot run" \ + "${LaneRelevance}" + return 1 + fi + echo "building the rocKE C++ engine extension (${Root})" + if ! cmake -S "${ROCKE_TOP}" -B "${Root}" -DCMAKE_BUILD_TYPE=Release \ + -DROCKE_BUILD_PYBIND=ON -Dpybind11_DIR="${PyBind}" \ + -DPython3_EXECUTABLE="${PyBin}" > "${Root}/configure.log" 2>&1; then + rockeResult setup engine-extension 1 \ + "cmake configure failed for the engine extension (see ${Root}/configure.log)" \ + "${LaneRelevance}" + return 1 + fi + if ! cmake --build "${Root}" --target rocke_core rocke_engine \ + -j "$(nproc)" > "${Root}/build.log" 2>&1; then + # The COD compiles this, so a failure here is a genuine COD finding, not noise. + rockeResult setup engine-extension 1 \ + "COD build of the engine extension failed (see ${Root}/build.log)" \ + "${LaneRelevance}" + return 1 + fi + Ext="$(find "${Root}" -name 'rocke_engine*.so' -print -quit 2>/dev/null)" + if [[ -z "${Ext}" ]]; then + rockeResult setup engine-extension 1 \ + "engine extension not produced under ${Root}" "${LaneRelevance}" + return 1 + fi + EngineExtDir="$(dirname "${Ext}")" +} + +function runPytest { # [pytest args...] + local Xml="${1}" Manifest="${2}" Log="${3}"; shift 3 + # A reused build dir (ROCKE_REBUILD=0) still holds the previous report; if + # this run never writes one, emitJunit would republish it as today's verdict. + rm -f "${Xml}" "${Manifest}" "${Log}" + ( cd "${ROCKE_TOP}/tests" \ + && ROCKE_RELEVANCE_OUT="${Manifest}" \ + PYTHONPATH="${PYTHONPATH}:${HelperDir}${EngineExtDir:+:${EngineExtDir}}" \ + "${PyBin}" -m pytest "$@" -p rocke_relevance -q --junitxml="${Xml}" ) 2>&1 \ + | tee "${Log}" + return "${PIPESTATUS[0]}" +} + +# Turn a JUnit report into result rows, or emit a red row when it is missing. +# Preserve an unexplained nonzero runner exit even when it left a partial XML +# containing only completed, passing testcases. +function emitJunit { # [runner-status] [manifest] [runner-log] + local Xml="${1}" GroupDefault="${2}" RunStatus="${3:-0}" Manifest="${4:-}" + local RunnerLog="${5:-}" + local ParseStatus=0 + local ExpectedFailureStatus=1 + [[ "${GroupDefault}" == ctest ]] && ExpectedFailureStatus=8 + local -a RelevanceArgs=(--relevance-default "${LaneRelevance}") + [[ -n "${Manifest}" && -f "${Manifest}" ]] \ + && RelevanceArgs+=(--relevance "${Manifest}") + # A lane whose every test drives the toolchain by construction says so, so a row + # cannot be reported below that even when the evidence is empty (work in a child). + [[ -n "${LaneRelevanceFloor:-}" ]] \ + && RelevanceArgs+=(--relevance-floor "${LaneRelevanceFloor}") + if [[ -f "${Xml}" ]]; then + "${PyBin}" "${HelperDir}/rocke_junit_results.py" \ + --junit "${Xml}" --group-default "${GroupDefault}" \ + "${RelevanceArgs[@]}" || ParseStatus=$? + if (( ParseStatus != 0 )); then + rockeResult setup "${GroupDefault}-junit" 1 \ + "cannot parse JUnit report (status ${ParseStatus})" + fi + if (( RunStatus != 0 )) \ + && { (( RunStatus != ExpectedFailureStatus )) \ + || ! grep -Eq '<(failure|error)[ />]' "${Xml}"; }; then + rockeResult setup "${GroupDefault}-runner" 1 \ + "test runner exited with status ${RunStatus}$(runnerDetail "${RunnerLog}")" + fi + else + rockeResult setup "${GroupDefault}-report" 1 \ + "no JUnit report produced$(runnerDetail "${RunnerLog}")" + fi +} + +# Reuse an existing venv, else create one (numpy + pytest) outside the source +# tree; fall back to the system python only if the venv cannot be built. +function setupPython { + local Need='import numpy, pytest' Existed=1 + PyBin="" + if [[ -x "${ROCKE_VENV}/bin/python" ]]; then + PyBin="${ROCKE_VENV}/bin/python" + # An interrupted or offline bootstrap leaves a venv that every later run + # would adopt and then fail on; top it up instead of inheriting the damage, + # unless the caller asked this script to install nothing. + if [[ "${ROCKE_SETUP_VENV}" == "1" ]] && ! "${PyBin}" -c "${Need}" 2>/dev/null; then + "${PyBin}" -m pip install --quiet numpy pytest || true + fi + elif [[ "${ROCKE_SETUP_VENV}" == "1" ]]; then + [[ -e "${ROCKE_VENV}" ]] || Existed=0 + if python3 -m venv "${ROCKE_VENV}" \ + && "${ROCKE_VENV}/bin/python" -m pip install --quiet --upgrade pip \ + && "${ROCKE_VENV}/bin/python" -m pip install --quiet numpy pytest; then + PyBin="${ROCKE_VENV}/bin/python" + elif (( Existed == 0 )); then + rm -rf "${ROCKE_VENV}" # only what this run created + fi + fi + if [[ -z "${PyBin}" ]]; then + echo "# WARN: venv unavailable, falling back to system python3" + # Name the real problem here: an empty PyBin would fail the import check + # below and report a missing module instead of a missing interpreter. + PyBin="$(command -v python3)" \ + || fatalSetup "venv unavailable and no python3 in PATH" python + fi + # Print once: the 'all' children resolve the same PyBin and would only repeat it. + [[ "${InternalAllChild:-0}" == 1 ]] || echo "# PyBin=${PyBin}" + # pytest matters as much as numpy: without it a lane exits 1 with no report + # and the row blames a missing JUnit file. + "${PyBin}" -c "${Need}" 2>/dev/null \ + || fatalSetup "numpy and pytest must be importable with ${PyBin} (venv ${ROCKE_VENV})" python +} + +# Refuse to install into an interpreter this script did not create. Standalone, +# PyBin can be the engineer's system python3, and these are large packages that +# would stay behind in ~/.local long after the run. +function pipInstallable { # + [[ "${PyBin}" == "${ROCKE_VENV}/bin/"* ]] && return 0 + echo "WARNING: not installing ${1} into ${PyBin}: outside ${ROCKE_VENV}." \ + "Allow the venv (ROCKE_SETUP_VENV=1) or preinstall it yourself." + return 1 +} + +# Dependencies declared by rocKE's platform `[project.optional-dependencies].dev` +# that are needed by its package-local heuristics tests. Keep torch separate: it +# must match the GPU stack and is provisioned only by gpu-numeric. +# rocKE's own declared dev extras, so a dependency it adds arrives here without an +# edit. A hardcoded list silently rots: rocKE has declared pybind11 for a while and +# our list omitted it, which left every cross-engine test unable to run. +# ${ROCKE_EXTRA_TEST_DEPS} covers what rocKE uses but does not declare. +function rockeDeclaredTestDeps { + "${PyBin}" - "${ROCKE_TOP}/pyproject.toml" <<'PY' 2>/dev/null +import re, sys +try: + text = open(sys.argv[1], encoding="utf-8").read() +except OSError: + sys.exit(1) +block = re.search(r"^dev\s*=\s*\[(.*?)\]", text, re.M | re.S) +if not block: + sys.exit(1) +names = re.findall(r'"([A-Za-z0-9][A-Za-z0-9._-]*)', block.group(1)) +print(" ".join(dict.fromkeys(names))) +PY +} + +function ensureProjectTestDeps { + local -a Deps=() + read -ra Deps <<< "$(rockeDeclaredTestDeps) ${ROCKE_EXTRA_TEST_DEPS}" + if (( ${#Deps[@]} == 0 )); then + rockeResult setup python-test-deps 1 \ + "cannot read rocKE's declared dev dependencies from ${ROCKE_TOP}/pyproject.toml" + return 1 + fi + # Import names differ from distribution names often enough that checking them is + # its own maintenance burden; pip already decides in milliseconds when satisfied. + echo "provisioning rocKE-declared test dependencies: ${Deps[*]}" + if pipInstallable "rocKE dev test dependencies" \ + && "${PyBin}" -m pip install --quiet "${Deps[@]}" + then + return 0 + fi + rockeResult setup python-test-deps 1 "cannot provision rocKE dev test dependencies" + return 1 +} + +# COD ROCm major.minor, from this install's own metadata only: rocKE's comgr +# resolver falls back to /opt/rocm when COD metadata is absent, and that +# unrelated version would select an incompatible multi-gigabyte torch wheel. +function codRocmVersion { + head -1 "${RocmRoot}/.info/version" 2>/dev/null | cut -d. -f1,2 +} + +# IR flavor era a ROCm major.minor implies, by rocKE's own rule (>= 7.2 is llvm22). +# Empty when the version is unusable. +function rocmEra { # + local Major="${1%%.*}" Rest="${1#*.}" Minor=0 + # A bare major must not borrow itself as the minor: "7" is 7.0, not 7.7. + [[ "${1}" == *.* ]] && Minor="${Rest%%.*}" + [[ "${Major}" =~ ^[0-9]+$ && "${Minor}" =~ ^[0-9]+$ ]] || return 1 + if (( Major > 7 || (Major == 7 && Minor >= 2) )); then echo llvm22; else echo llvm20; fi +} + +# Accept a torch that can serve as the numeric reference for this COD. +# +# Not an exact ROCm match: a COD carries an in-development ROCm (7.15, 10.0) that +# no published torch will ever match, so requiring equality makes the lane dead on +# every COD. What keeps the toolchain honest is ROCKE_COMGR_LIB, which pins rocKE +# to the COD comgr ahead of any torch-bundled one; torch's job here is to be a +# numeric oracle. So require only the same IR flavor era -- across eras its HIP +# runtime pairs badly with COD kernels -- and note a difference within one. +function validateTorch { + local Ver TorchVer TorchMajorMinor CodEra TorchEra + "${PyBin}" -c 'import torch' 2>/dev/null || return 1 + TorchVer="$("${PyBin}" -c 'import torch; print(torch.version.hip or "")' 2>/dev/null)" + [[ -n "${TorchVer}" ]] || { + echo "ERROR: ROCKE_VENV contains a non-ROCm torch build" + return 1 + } + TorchMajorMinor="$(cut -d. -f1,2 <<< "${TorchVer}")" + Ver="$(codRocmVersion)" + CodEra="$(rocmEra "${Ver}" || true)" + TorchEra="$(rocmEra "${TorchMajorMinor}" || true)" + if [[ -n "${CodEra}" && -n "${TorchEra}" && "${CodEra}" != "${TorchEra}" ]]; then + echo "ERROR: torch ROCm ${TorchVer} is ${TorchEra}, but COD ROCm ${Ver} is ${CodEra}" + return 1 + fi + if [[ -n "${Ver}" && "${TorchMajorMinor}" != "${Ver}" ]]; then + echo "using torch ROCm ${TorchVer} as the numeric reference for COD ROCm ${Ver}" \ + "(same ${CodEra} era; rocKE still compiles through the COD comgr)" + else + echo "using torch ROCm ${TorchVer} from ${ROCKE_VENV}" + fi + return 0 +} + +# Provision rocKE's numeric reference (torch) on demand from the pytorch ROCm +# wheel index for the COD's ROCm major.minor; ROCKE_TORCH_INDEX_URL overrides it. +function ensureTorch { + local Idx="${ROCKE_TORCH_INDEX_URL}" Ver + validateTorch && return 0 + if [[ -z "${Idx}" ]]; then + Ver="$(codRocmVersion)" + if [[ -z "${Ver}" ]]; then + echo "ERROR: COD has no .info/version; set ROCKE_TORCH_INDEX_URL or preinstall torch in ROCKE_VENV" + return 1 + fi + # A guess, and often a wrong one: pytorch publishes an index per *released* + # ROCm, and a COD carries an unreleased one. The failure path below says what + # to do about it. + Idx="https://download.pytorch.org/whl/rocm${Ver}" + fi + pipInstallable "torch (multiple GB)" || return 1 + # A wheel for the wrong ROCm still satisfies the requirement, so a plain + # install would be a no-op here and validateTorch would fail again. + local -a Force=() + "${PyBin}" -c 'import torch' 2>/dev/null && Force=(--force-reinstall) + echo "provisioning torch for gpu-numeric from ${Idx}" + "${PyBin}" -m pip install "${Force[@]}" --index-url "${Idx}" torch || { + echo "ERROR: no torch at ${Idx}" + echo " An unreleased COD ROCm has no published torch. Set" \ + "ROCKE_TORCH_INDEX_URL to a released" + echo " index of the same era ($(rocmEra "$(codRocmVersion)" || echo '?'))," \ + "e.g. https://download.pytorch.org/whl/rocm7.2," + echo " or preinstall torch in ${ROCKE_VENV}." + return 1 + } + validateTorch +} + +# branch@shortsha of the rocKE checkout, or '?' when it is not a git tree. +function rockeSrcRev { + local Branch Sha + Branch="$(git -C "${ROCKE_TOP}" rev-parse --abbrev-ref HEAD 2>/dev/null || echo '?')" + Sha="$(git -C "${ROCKE_TOP}" rev-parse --short HEAD 2>/dev/null || echo '?')" + echo "${Branch}@${Sha}" +} + +# Ensure the shared rocm-libraries checkout is usable before testing. rocKE and CK +# share this tree (${AOMP_REPOS_TEST}/composable-kernels/rocm-libraries), so a +# refresh must never discard another user's work: clone only into a missing/empty +# path, require an existing checkout to be clean, fetch the requested branch, and +# advance its local branch by fast-forward only. +function updateRockeSource { + local Top Origin SourceLock + local Repo="${ROCKE_REPO_ROOT}" + local Url="${ROCKE_REPO_URL}" + local Branch="${ROCKE_REPO_BRANCH}" + if [[ "${Repo}" != "${ROCKE_TOP}" ]]; then + mkdir -p "$(dirname "${Repo}")" \ + || fatalSetup "cannot create source parent: $(dirname "${Repo}")" source + SourceLock="${Repo}.rocke-ci.lock.d" + acquireDirLock "${SourceLock}" "rocKE source" + fi + if [[ "${Repo}" == "${ROCKE_TOP}" ]]; then + echo "WARN: cannot derive the rocm-libraries root from a custom ROCKE_TOP" + elif ! git -C "${Repo}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + if [[ -d "${Repo}" && -n "$(find "${Repo}" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then + fatalSetup "refusing to replace non-empty non-git source path: ${Repo}" source + fi + echo "no rocKE checkout under ${ROCKE_TOP}; cloning rocm-libraries (several GB," + echo "shared with CK) into ${Repo}. Point ROCKE_TOP at an existing checkout to skip." + rmdir "${Repo}" 2>/dev/null || true + git clone --single-branch --depth 1 -b "${Branch}" "${Url}" "${Repo}" \ + || fatalSetup "rocm-libraries clone failed: ${Url} (${Branch})" source + else + Top="$(realpath -m "$(git -C "${Repo}" rev-parse --show-toplevel)")" + [[ "${Top}" == "$(realpath -m "${Repo}")" ]] \ + || fatalSetup \ + "source path is nested in another repository (${Top}); refusing to update: ${Repo}" \ + source + if [[ "${ROCKE_UPDATE_REPO}" == 1 ]]; then + git check-ref-format --branch "${Branch}" >/dev/null 2>&1 \ + || fatalSetup "invalid ROCKE_REPO_BRANCH: ${Branch}" source + Origin="$(git -C "${Repo}" remote get-url origin 2>/dev/null || true)" + [[ "${Origin}" == "${Url}" ]] \ + || fatalSetup \ + "source origin mismatch: expected ${Url}, found ${Origin:-}" \ + source + if [[ -n "$(git -C "${Repo}" status --porcelain)" ]]; then + fatalSetup \ + "rocm-libraries checkout has local changes; refusing to update: ${Repo}" \ + source + fi + echo "updating rocm-libraries (${Repo})" + git -C "${Repo}" fetch --prune origin \ + "+refs/heads/${Branch}:refs/remotes/origin/${Branch}" \ + || fatalSetup "failed to fetch origin/${Branch}; refusing to test stale source" source + if git -C "${Repo}" show-ref --verify --quiet "refs/heads/${Branch}"; then + git -C "${Repo}" merge-base --is-ancestor "${Branch}" "origin/${Branch}" \ + || fatalSetup \ + "local ${Branch} is not a fast-forward of origin/${Branch}; refusing to rewrite it" \ + source + git -C "${Repo}" switch "${Branch}" \ + || fatalSetup "failed to switch to source branch ${Branch}" source + else + git -C "${Repo}" switch --track -c "${Branch}" "origin/${Branch}" \ + || fatalSetup "failed to create tracking branch ${Branch}" source + fi + git -C "${Repo}" merge --ff-only "origin/${Branch}" \ + || fatalSetup "failed to fast-forward ${Branch}; refusing to test stale source" source + fi + fi + echo "rocKE src = $(rockeSrcRev) (rocm-libraries: ${ROCKE_TOP})" +} + +function printBanner { + local ClangVer LlvmSha HipVer + ClangVer="$("${CXX}" --version 2>/dev/null | head -1)" + # The COD clang embeds its llvm-project git SHA in --version; grab it so a stale + # COD (or a re-tagged same-SHA build) is identifiable from the log alone. + LlvmSha="$("${CXX}" --version 2>/dev/null | grep -oE '[0-9a-f]{12,40}' | tail -1)" + HipVer="$(awk -F= ' + /^HIP_VERSION_(MAJOR|MINOR|PATCH|GITHASH)=/ { v[$1] = $2 } + END { if (v["HIP_VERSION_MAJOR"] != "") + printf "%s.%s.%s-%s", v["HIP_VERSION_MAJOR"], v["HIP_VERSION_MINOR"], \ + v["HIP_VERSION_PATCH"], v["HIP_VERSION_GITHASH"] } + ' "${RocmRoot}/share/hip/version" 2>/dev/null)" + echo "===============================================================================" + echo "rocKE ${Stage} ($(date '+%Y-%m-%d %H:%M:%S'))" + echo " AOMP = ${AompInput} -> ${AOMP}" + echo " ROCM_PATH = ${ROCM_PATH} (${RocmRootSource})" + echo " clang = ${ClangVer}" + echo " llvm SHA = ${LlvmSha:-?}" + echo " HIP = ${HipVer:-?}" + echo " flavors = codegen:${ROCKE_CODEGEN_FLAVOR} comgr:${ROCKE_COMGR_FLAVOR} engine:${ROCKE_ENGINE_FLAVORS}" + echo "===============================================================================" +} + +# True when a resolved path lives inside the COD install root. +function underCod { + [[ -n "${1}" && -e "${1}" && "$(realpath -m "${1}")" == "${RocmRoot}"/* ]] +} + +# Print one hygiene row; return non-zero when a *hard* requirement is external. +function codToolchainRow { #