Source code for grid_rbd._artifacts

"""Consumer artifacts: a caller's CUDA sources compiled against a generated `grid.cuh` under
the store's rules (HJCD-IK ask G9, 2026-10-23).

A consumer such as an IK or trajectory-optimization solver builds one kernel translation unit
per robot against the header GRiD generates for that robot. `build_artifact` runs that build
the way `register_robot` builds its own wrapper: the sources are generated into a staging
directory, hashed with every compile input into a content key, compiled only if that key has
no artifact yet (each unit from the object cache when its exact inputs were compiled before),
and published atomically under a namespace of the cache directory that belongs to the
consumer:

    <cache_dir>/consumers/<consumer>/store/<content_key>/   grid.cuh, <artifact_name>, meta.json,
                                                            build_inputs.json, build.log
    <cache_dir>/consumers/<consumer>/bykey/<input_key>      pointer from the request to the content
    <cache_dir>/consumers/<consumer>/objects/               per-unit objects, keyed by their inputs
    <cache_dir>/consumers/<consumer>/manifest.json          name -> content key, under a lock

The identity covers the URDF bytes, the codegen options, the codegen tree, the consumer's
sources and headers by content, the defines and flags, the toolchain, the CUDA architecture
and the package version; a pointer is honoured only when the identity recorded beside the
artifact equals the current one. `load_artifact` returns a prepared artifact without a
toolchain, refusing one recorded for another architecture or key schema. The caller loads the
artifact itself (dlopen, ctypes, its own binding); GRiD never binds the consumer's symbols.
"""
from __future__ import annotations

import concurrent.futures
import fcntl
import hashlib
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import sysconfig
import threading
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Sequence

from . import _cache
from ._cache import (_KEY_SCHEMA, _arch_loadable, _sweep_dead_staging_dirs, canonical_options,
                     default_cache_dir, detect_cuda_arch, effective_cuda_archs, keymap_lookup,
                     keymap_record, package_version, read_build_inputs, stale_hit_reasons,
                     write_build_inputs)
from ._compile import _NVCC_DEFAULT_FLAGS, _build_jobs, find_nvcc, generate_grid_cuh
from ._store import entry_metadata, publish_entry, publish_object, staging_dir
from ._consumer_toolchain import resolve_consumer_toolchain
from ._consumer_dependencies import discover_dependencies
from ._model_artifact import (check_expected_id, file_digest, record_model_association,
                              validate_model_association)

_log = logging.getLogger("grid_rbd")

ARTIFACT_SCHEMA = 2
_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
# What a load must match: key schema, device, and the ABI for Python extensions. The
# toolchain, GLASS content and the codegen tree are provenance, recorded but not required to
# load a prepared artifact on a box that never had them.
LOAD_IDENTITY_KEYS = ("key_schema", "artifact_schema", "cuda_arch", "python_abi")
_SOURCE_SUFFIXES = (".cu", ".cpp", ".cc", ".cxx")


[docs] class ConsumerArtifactError(RuntimeError): """Base of the artifact errors."""
[docs] class ArtifactNotFoundError(ConsumerArtifactError, KeyError): """No artifact of that name in the consumer's manifest."""
[docs] class ArtifactIncompatibleError(ConsumerArtifactError): """The recorded artifact cannot be loaded here (another device or key schema)."""
[docs] @dataclass(frozen=True) class ConsumerArtifact: consumer: str name: str path: Path # the shared object header: Path # the generated grid.cuh it was compiled against store_dir: Path content_key: str input_key: str | None cuda_arch: int meta: dict[str, Any] compiled: bool # nvcc ran in this call (False: reused or loaded) @property def model_descriptor(self): """Validated coordinate metadata, not complete dynamics/artifact identity.""" try: return validate_model_association(self.store_dir, self.meta, self.path.name) except ValueError as exc: raise ArtifactIncompatibleError(str(exc)) from exc
# ─── paths ────────────────────────────────────────────────────────────────── def consumer_root(cache_dir: Path, consumer: str) -> Path: return cache_dir / "consumers" / consumer def _check_name(value: str, what: str) -> str: if not isinstance(value, str) or not _NAME_RE.match(value): raise ValueError(f"{what} must be a short identifier (letters, digits, '_', '.', '-'), got {value!r}") return value def _artifact_module_hash() -> str: try: h = hashlib.sha256(Path(__file__).read_bytes()) h.update(Path(__file__).with_name("_consumer_toolchain.py").read_bytes()) h.update(Path(__file__).with_name("_consumer_dependencies.py").read_bytes()) h.update(Path(__file__).with_name("_model_artifact.py").read_bytes()) return h.hexdigest() except OSError: return "" def _collision_include_dir() -> Path: import grid_codegen return Path(grid_codegen.__file__).resolve().parent / "collision" def _glass_include_dirs() -> list[Path]: from grid_codegen.resources import resource_path root = Path(resource_path("GLASS")) return [root, root / "src"] def artifact_identity(cuda_arch: int) -> dict[str, Any]: """Every artifact-shaping input that is not a caller argument, recorded beside each entry.""" identity: dict[str, Any] = { "key_schema": _KEY_SCHEMA, "artifact_schema": ARTIFACT_SCHEMA, "cuda_arch": int(cuda_arch), "nvcc": _cache._nvcc_identity(), "host_cxx": _cache._host_cxx_tag(), "glass_content": _cache._glass_content_hash(), "glass_commit": _cache._glass_tag(), "codegen_tree": _cache._codegen_source_hash(), "compile_py": _cache._compile_source_hash(), "artifacts_py": _artifact_module_hash(), "grid_rbd": package_version(), "generation_env": _cache._generation_env(), } archs = effective_cuda_archs(cuda_arch) if archs is not None: identity["cuda_archs"] = archs return identity def _python_abi_identity() -> dict[str, Any]: """Interpreter ABI for declared extensions, without querying a compiler.""" return { "implementation": sys.implementation.name, "version": list(sys.version_info[:2]), "cache_tag": sys.implementation.cache_tag, "soabi": sysconfig.get_config_var("SOABI"), "ext_suffix": sysconfig.get_config_var("EXT_SUFFIX"), "debug": bool(sysconfig.get_config_var("Py_DEBUG")), "free_threaded": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), } def _validate_python_exports(path: Path, python_extension: bool) -> None: """Validate the pre-lookup declaration; inspecting a binary alone is too late to protect reuse. Consumer builds require nm, prepared loads do not.""" result = subprocess.run(["nm", "-D", "--defined-only", str(path)], capture_output=True, text=True, check=True) has_pyinit = any(line.split() and line.split()[-1].startswith("PyInit_") for line in result.stdout.splitlines()) if has_pyinit != python_extension: raise ValueError("python_extension declaration disagrees with exported PyInit_* symbols; " "pass python_extension=True for a Python extension and False for a native library") def _collision_mesh_identity(urdf_path: Path) -> list[dict[str, str]]: """Resolve with the spherizer's rules before considering a fast cache hit.""" from grid_codegen.algorithms._spherize import _resolve_mesh_path dependencies = [] for link in ET.parse(urdf_path).getroot().findall("link"): for mesh in link.findall("collision/geometry/mesh"): filename = mesh.get("filename", "") resolved = _resolve_mesh_path(filename, str(urdf_path.parent)) if resolved is None: raise ValueError(f"collision mesh {filename!r} for link {link.get('name')!r} " "could not be resolved; provide the asset locally") path = Path(resolved).resolve() dependencies.append({"uri": filename, "path": str(path), "sha256": file_digest(path)}) return dependencies def _resolve_files(values: Sequence[str | Path], what: str, suffixes: tuple[str, ...] | None) -> list[Path]: out: list[Path] = [] for v in values: p = Path(v).expanduser().resolve() if not p.is_file(): raise FileNotFoundError(f"{what} not found: {p}") if suffixes is not None and p.suffix not in suffixes: raise ValueError(f"{what} {p.name}: expected one of {suffixes}") out.append(p) if len({p.name for p in out}) != len(out): raise ValueError(f"{what} must have distinct file names (each becomes one object): {[p.name for p in out]}") return out def _normalize_defines(defines: Sequence[str] | dict[str, Any] | None) -> list[str]: if not defines: return [] if isinstance(defines, dict): items = [k if v is None else f"{k}={v}" for k, v in defines.items()] else: items = [str(d) for d in defines] for d in items: if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*(=.*)?$", d): raise ValueError(f"define {d!r} is not NAME or NAME=VALUE") return sorted(items) # ─── code options (the subset of register_robot's that shape grid.cuh) ──────── def _code_options(urdf_path: Path, cuda_arch: int, *, floating_base, ee_joint_names, dtype, algorithm_list, enable_mujoco_kernels, cost_frames, contact_frames, use_joint_dynamics, friction_model, friction_width, runtime_inertia, runtime_transform, runtime_joint_dynamics, emit_plant, collision, collision_res, collision_native, spherized_urdf, strict_collision, launch_config_profile, runtime_armature: bool = False, armature=None) -> dict[str, Any]: if dtype not in ("float32", "float64"): raise ValueError(f"dtype must be 'float32' or 'float64', got {dtype!r}") opts: dict[str, Any] = { "floating_base": bool(floating_base), "ee_joint_names": list(ee_joint_names) if ee_joint_names else [], "cuda_arch": int(cuda_arch), } if dtype == "float64": opts["dtype"] = "float64" if algorithm_list is not None: if isinstance(algorithm_list, str): algos = [a.strip() for a in algorithm_list.replace(";", ",").split(",") if a.strip()] else: algos = [str(a).strip() for a in algorithm_list if str(a).strip()] if not algos: raise ValueError("algorithm_list must name at least one algorithm or profile") opts["algorithm_list"] = sorted(set(algos)) if not enable_mujoco_kernels: opts["enable_mujoco_kernels"] = False if cost_frames: opts["cost_frames"] = [str(n) for n in cost_frames] if contact_frames: opts["contact_frames"] = [str(n) for n in contact_frames] if use_joint_dynamics: opts["use_joint_dynamics"] = True if friction_model not in ("sign", "tanh"): raise ValueError(f"friction_model must be 'sign' or 'tanh', got {friction_model!r}") if friction_model == "tanh" or friction_width is not None: if not use_joint_dynamics: raise ValueError("friction_model='tanh' / friction_width require use_joint_dynamics=True") if friction_model != "tanh": raise ValueError("friction_width applies to friction_model='tanh' only") opts["friction_model"] = "tanh" if friction_width is not None: try: opts["friction_width"] = [float(w) for w in friction_width] except TypeError: opts["friction_width"] = float(friction_width) for flag, value in (("runtime_inertia", runtime_inertia), ("runtime_transform", runtime_transform), ("runtime_joint_dynamics", runtime_joint_dynamics), ("runtime_armature", runtime_armature)): if value: opts[flag] = True if armature is not None: from grid_codegen.armature import option_vector opts["armature"] = option_vector(armature) if emit_plant is not None: opts["emit_plant"] = bool(emit_plant) else: opts["emit_plant"] = None # the generator's rule (E1) decides; recorded so it is keyed if sum(bool(x) for x in (collision, collision_native, spherized_urdf)) > 1: raise ValueError("pass one of collision=True, collision_native=True or spherized_urdf=") if spherized_urdf: sp = Path(spherized_urdf).expanduser().resolve() if not sp.is_file(): raise FileNotFoundError(f"spherized_urdf not found: {sp}") opts["spherized_urdf"] = str(sp) opts["spherized_urdf_sha256"] = file_digest(sp) # the key follows the model's content elif collision_native: opts["collision_native"] = True opts["collision_res"] = float(collision_res) if not isinstance(collision_res, (list, tuple)) \ else [float(r) for r in collision_res] elif collision: opts["collision"] = True opts["collision_res"] = float(collision_res) if not isinstance(collision_res, (list, tuple)) \ else [float(r) for r in collision_res] if collision or collision_native: opts["collision_meshes"] = _collision_mesh_identity(urdf_path) if not strict_collision: opts["strict_collision"] = False elif not strict_collision: raise ValueError("strict_collision=False applies to the spherizer (collision=True or " "collision_native=True); a supplied spherized_urdf is validated, not spherized") if launch_config_profile is not None: opts["launch_config_profile"] = str(launch_config_profile) # The baked per-algorithm launch configuration is not derivable from the URDF bytes; fold # the resolved values in so a re-autotune re-keys the artifact (as register_robot does). try: from grid_codegen.GRiDCodeGenerator import load_launch_config from grid_codegen.launch_config import select_launch_config_gpu from ._compile import _resolve_launch_config_robot lc_robot = _resolve_launch_config_robot(str(urdf_path)) lc_gpu = select_launch_config_gpu(lc_robot, cuda_arch) opts["launch_config_gpu"] = lc_gpu lc = load_launch_config(lc_robot, bool(floating_base), gpu=lc_gpu, profile=opts.get("launch_config_profile", "ffi")) if lc: opts["launch_config"] = {k: dict(v) for k, v in sorted(lc.items())} except Exception: pass return opts # ─── keys ─────────────────────────────────────────────────────────────────── def _input_key(urdf_bytes: bytes, code_options: dict[str, Any], sources: list[Path], headers: list[Path], include_dirs: list[Path], defines: list[str], extra_flags: list[str], link_flags: list[str], artifact_name: str, identity: dict[str, Any]) -> str: h = hashlib.sha256() h.update(urdf_bytes) h.update(canonical_options(code_options).encode()) for p in sources: h.update(b"source:" + str(p).encode() + b"\0" + p.read_bytes() + b"\0") for p in headers: h.update(b"header:" + p.name.encode() + b"\0" + p.read_bytes() + b"\0") h.update(("include_dirs=" + ";".join(str(d) for d in include_dirs)).encode()) h.update(("defines=" + ";".join(defines)).encode()) h.update(("extra_flags=" + ";".join(extra_flags)).encode()) h.update(("link_flags=" + ";".join(link_flags)).encode()) h.update(("artifact_name=" + artifact_name).encode()) h.update(("identity=" + canonical_options(identity)).encode()) return h.hexdigest() def _toolchain_tag() -> str: return (f"nvcc={_cache._nvcc_identity()};host_cxx={_cache._host_cxx_tag()};" f"glass={_cache._glass_content_hash()};key_schema={_KEY_SCHEMA};artifact_schema={ARTIFACT_SCHEMA}") def _hash_build_context(h, python_abi, toolchain, dependencies) -> None: """The declarations both build keys open with: the Python ABI, the consumer toolchain identity and the dependency capture, each canonically serialized.""" h.update(("python_abi=" + canonical_options(python_abi)).encode()) h.update(("consumer_toolchain=" + canonical_options(toolchain)).encode()) h.update(("dependencies=" + canonical_options(dependencies)).encode()) def _hash_compile_flags(h, compile_cmd: list[str], gen_dir: Path) -> None: """Every nvcc argument except the staging directory's own paths, whose content is hashed.""" src = str(gen_dir) for a in compile_cmd[1:]: if a == f"-I{src}" or a.startswith(src + "/"): continue h.update(a.encode() + b"\0") def _content_key(gen_dir: Path, sources: list[Path], headers: list[Path], compile_cmd: list[str], link_flags: list[str], artifact_name: str, *, python_abi, toolchain, dependencies) -> str: """Exactly what nvcc sees: the generated header, the static collision header, the consumer's sources and headers, the flag list (the staging dir's own -I removed; its content is hashed), the link flags and the toolchain.""" h = hashlib.sha256() _hash_build_context(h, python_abi, toolchain, dependencies) h.update((gen_dir / "grid.cuh").read_bytes()) h.update(_collision_include_dir().joinpath("grid_collision_geometry.cuh").read_bytes()) for p in sources: h.update(b"source:" + str(p).encode() + b"\0" + p.read_bytes() + b"\0") for p in headers: h.update(b"header:" + p.name.encode() + b"\0" + p.read_bytes() + b"\0") _hash_compile_flags(h, compile_cmd, gen_dir) h.update(("link_flags=" + ";".join(link_flags) + ";artifact_name=" + artifact_name).encode()) h.update(_toolchain_tag().encode()) return h.hexdigest() def _unit_object_key(unit: Path, gen_dir: Path, headers: list[Path], compile_cmd: list[str], *, python_abi, toolchain, dependencies) -> str: h = hashlib.sha256() _hash_build_context(h, python_abi, toolchain, dependencies) # nvcc receives this exact resolved path: suffix selects the language and # __FILE__ exposes the whole path, not merely its basename. h.update(str(unit).encode() + b"\0") _hash_compile_flags(h, compile_cmd, gen_dir) h.update((gen_dir / "grid.cuh").read_bytes()); h.update(b"\0") h.update(_collision_include_dir().joinpath("grid_collision_geometry.cuh").read_bytes()); h.update(b"\0") for p in headers: h.update(p.read_bytes()); h.update(b"\0") h.update(unit.read_bytes()) h.update(_toolchain_tag().encode()) return h.hexdigest() # ─── the compile half ───────────────────────────────────────────────────────── def _compile_command(gen_dir: Path, cuda_arch: int, include_dirs: list[Path], defines: list[str], toolchain) -> list[str]: """The nvcc command of every consumer unit; the toolchain carries the consumer's extra flags, the host compiler selection and the symbol isolation flags.""" nvcc = find_nvcc() archs = effective_cuda_archs(cuda_arch) gencode_archs = archs if archs is not None else [cuda_arch] cmd = [nvcc] + toolchain.prepend + list(_NVCC_DEFAULT_FLAGS) cmd += [f"-gencode=arch=compute_{a},code=sm_{a}" for a in gencode_archs] cmd.append(f"-I{gen_dir}") cmd.append(f"-I{_collision_include_dir()}") for d in _glass_include_dirs(): cmd.append(f"-I{d}") for d in include_dirs: cmd.append(f"-I{d}") for d in defines: cmd.append(f"-D{d}") cmd.extend(toolchain.compile_flags()) return cmd def _build(gen_dir: Path, sources: list[Path], headers: list[Path], compile_cmd: list[str], link_flags: list[str], out_so: Path, object_cache_dir: Path, reuse_objects: bool, *, validate_unit, python_abi, toolchain, dependencies) -> dict[str, Any]: """Compile each consumer source with -c (from the object cache when its key exists), then host-link the objects into `out_so`. Returns a record of what ran. `validate_unit(unit)` runs before a unit is compiled and before its object is published; it raises when the build inputs or that unit's dependencies changed.""" nvcc = compile_cmd[0] jobs = _build_jobs() build_dir = gen_dir / "build" build_dir.mkdir(exist_ok=True) object_cache_dir.mkdir(parents=True, exist_ok=True) log_path = gen_dir / "build.log" lock = threading.Lock() with log_path.open("w") as log: log.write("$ " + " ".join(compile_cmd) + " -c <unit> -o <unit>.o # x" + str(len(sources)) + "\n") log.write("$ " + " ".join([nvcc, "--shared", "-o", str(out_so), "<units>.o"] + link_flags) + "\n\n") record: dict[str, Any] = {"units": {}} def one(unit: Path) -> tuple[Path, int, float, str]: validate_unit(unit) key = _unit_object_key(unit, gen_dir, headers, compile_cmd, python_abi=python_abi, toolchain=toolchain.identity, dependencies=dependencies["units"][str(unit)]) cached = object_cache_dir / f"{unit.name}-{key}.o" if reuse_objects and cached.is_file(): os.utime(cached) with lock, log_path.open("a") as log: log.write(f"[{unit.stem}] cached {cached.name}\n") record["units"][unit.name] = {"object": cached.name, "compiled": False} return cached, 0, 0.0, "" obj = build_dir / (unit.name + ".o") ulog = build_dir / (unit.name + ".log") t0 = time.monotonic() with ulog.open("w") as f: r = subprocess.run(compile_cmd + ["-c", str(unit), "-o", str(obj)], stdout=f, stderr=subprocess.STDOUT, env=toolchain.environment()) secs = time.monotonic() - t0 with lock, log_path.open("a") as log: log.write(f"[{unit.stem}] rc={r.returncode} {secs:.1f}s\n") if r.returncode == 0: # Reject changed inputs before exposing an object under its old key, # not merely before publishing the final linked artifact. validate_unit(unit) publish_object(obj, cached) obj = cached record["units"][unit.name] = {"object": cached.name, "compiled": True, "seconds": round(secs, 1)} return obj, r.returncode, secs, ulog.read_text() with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as ex: results = list(ex.map(one, sources)) failed = [(obj, rc, out) for obj, rc, _, out in results if rc != 0] if failed: obj, rc, out = failed[0] raise RuntimeError(f"nvcc exit {rc} compiling the consumer unit {obj.stem}:\n{out}") objs = [str(obj) for obj, _, _, _ in results] link = toolchain.link_command(nvcc, out_so, objs) t0 = time.monotonic() with log_path.open("a") as log: log.write("\n$ " + " ".join(link) + "\n") log.flush() r = subprocess.run(link, stdout=log, stderr=subprocess.STDOUT, env=toolchain.environment()) log.write(f"[link] rc={r.returncode} {time.monotonic() - t0:.1f}s\n") if r.returncode != 0: raise RuntimeError(f"nvcc exit {r.returncode} linking the consumer artifact; build log:\n" + log_path.read_text()) shutil.rmtree(build_dir, ignore_errors=True) record["compiled_units"] = sum(1 for _, _, secs, _ in results if secs > 0) return record # ─── manifest (one per consumer, under a lock) ─────────────────────────────── def _schema_ok(record: dict[str, Any]) -> bool: """The record declares this artifact schema as an int (not a bool or a string).""" return type(record.get('artifact_schema')) is int and record['artifact_schema'] == ARTIFACT_SCHEMA def _abi_disagreement(meta: dict[str, Any], recorded: dict[str, Any] | None, python_extension: bool, python_abi) -> str | None: """Why the metadata or the identity record does not declare the requested extension ABI, or None when both agree with it.""" if (meta.get("python_extension") is not python_extension or "python_abi" not in meta or meta["python_abi"] != python_abi): return "python_abi: metadata declaration disagrees with requested artifact" if recorded is not None and "python_abi" not in recorded: return "python_abi: no extension declaration recorded" if recorded is not None and recorded["python_abi"] != python_abi: return "python_abi: identity record disagrees with requested artifact" return None def _manifest_path(root: Path) -> Path: return root / "manifest.json" def _load_manifest(root: Path) -> dict[str, Any]: path = _manifest_path(root) if not path.exists(): return {"artifact_schema": ARTIFACT_SCHEMA, "artifacts": {}} data = json.loads(path.read_text()) if data.get("artifact_schema") != ARTIFACT_SCHEMA: raise ConsumerArtifactError( f"{path} was written with artifact schema {data.get('artifact_schema')!r}; this grid_rbd expects " f"{ARTIFACT_SCHEMA}. Point cache_dir= at a fresh directory and build the artifacts again.") return data def _manifest_register(root: Path, name: str, content_key: str, meta: dict[str, Any]) -> None: root.mkdir(parents=True, exist_ok=True) with (root / "manifest.lock").open("a+") as lock: fcntl.flock(lock.fileno(), fcntl.LOCK_EX) try: manifest = _load_manifest(root) manifest["artifacts"][name] = {"content_key": content_key, **meta} tmp = _manifest_path(root).with_suffix(f".json.{os.getpid()}.tmp") tmp.write_text(json.dumps(manifest, indent=2, sort_keys=True)) tmp.replace(_manifest_path(root)) finally: fcntl.flock(lock.fileno(), fcntl.LOCK_UN) # ─── public operations ───────────────────────────────────────────────────────
[docs] def build_artifact(consumer: str, name: str, urdf_path: str | Path, *, sources: Sequence[str | Path], headers: Sequence[str | Path] = (), include_dirs: Sequence[str | Path] = (), defines: Sequence[str] | dict[str, Any] | None = None, extra_flags: Sequence[str] = (), link_flags: Sequence[str] = (), artifact_name: str = "artifact.so", python_extension: bool = False, expected_descriptor_id: str | None = None, cache_dir: str | Path | None = None, cuda_arch: int | None = None, force_rebuild: bool = False, plan_only: bool = False, floating_base: bool = False, ee_joint_names: Sequence[str] | None = None, dtype: str = "float32", algorithm_list: Sequence[str] | str | None = None, enable_mujoco_kernels: bool = True, cost_frames: Sequence[str] | None = None, contact_frames: Sequence[str] | None = None, use_joint_dynamics: bool = False, friction_model: str = "sign", friction_width=None, runtime_inertia: bool = False, runtime_transform: bool = False, runtime_joint_dynamics: bool = False, runtime_armature: bool = False, armature=None, emit_plant: bool | None = None, collision: bool = False, collision_res: float | Sequence[float] = 0.05, collision_native: bool = False, spherized_urdf: str | Path | None = None, strict_collision: bool = True, launch_config_profile: str | None = None) -> ConsumerArtifact | dict[str, Any]: """Build (or reuse) the consumer's artifact for one robot and return where it is. `consumer` names the namespace in the cache directory (one per consuming project); `name` the artifact within it (one per robot and configuration). `sources` are the consumer's translation units, each compiled with `-c` against the generated `grid.cuh` (plus the static collision header and the GLASS include directories) and linked with `--shared` into `artifact_name`. Non-system dependencies are discovered with nvcc before reuse/build and hashed by path and contents; `headers` supplements that discovery with explicitly declared dependencies. System headers excluded by -MM are an environment assumption, not covered by compiler identity alone. `defines` are `NAME` or `NAME=VALUE`; `extra_flags` go to every unit compile, `link_flags` to the link. Declare `python_extension=True` for Python/pybind extensions before cache lookup. Their interpreter ABI participates in pointer, content and object identity and is checked on prepared loads. Native C libraries (the default) are Python-independent. Builds check the declaration against exported PyInit_* symbols using `nm`. `expected_descriptor_id` optionally checks prior CPU model inspection on cold and reused builds. Schema-2 artifacts bind validated coordinate metadata to the generated header and binary; older artifacts need rebuilding. The codegen keywords (`floating_base`, `ee_joint_names`, `dtype`, `algorithm_list`, `enable_mujoco_kernels`, `cost_frames`, `contact_frames`, the joint-dynamics and runtime knobs) mean what they mean on `register_robot`; `emit_plant` is the generator's plant-block switch (None: the generator's rule), and `collision`, `collision_res`, `collision_native` and `spherized_urdf` select the generated `grid_collision` namespace (a sphere model is validated against the URDF). `strict_collision` (default True) makes the spherizer's warnings errors on the `collision` and `collision_native` routes: a mesh whose voxel fill failed and would be replaced by a bounding-box cover, or a model with no spheres, stops the build instead of silently changing the clearance rows; False accepts them with the warning. `launch_config_profile` (default: the bindings' "ffi" profile) names the baked launch-configuration profile. `plan_only=True` reports what a build would use without generating or compiling; validating an existing entry still invokes dependency preprocessing. Cold plans cannot know dependencies until a generated header exists. `force_rebuild=True` recompiles every unit. Prepared `load_artifact` requires neither source nor compiler.""" _check_name(consumer, "consumer") _check_name(name, "name") _check_name(artifact_name, "artifact_name") if not isinstance(python_extension, bool): raise TypeError("python_extension must be a bool") check_expected_id(expected_descriptor_id) urdf_p = Path(urdf_path).expanduser().resolve() if not urdf_p.is_file(): raise FileNotFoundError(f"URDF file not found: {urdf_p}") urdf_bytes = urdf_p.read_bytes() if expected_descriptor_id is not None: from ._model import describe_model actual = describe_model(urdf_p, floating_base=floating_base, runtime_transform=runtime_transform)['descriptor_id'] if actual != expected_descriptor_id: raise ValueError('expected_descriptor_id does not match requested model') srcs = _resolve_files(list(sources), "source", _SOURCE_SUFFIXES) if not srcs: raise ValueError("sources must name at least one translation unit") hdrs = _resolve_files(list(headers), "header", None) incs = [Path(d).expanduser().resolve() for d in include_dirs] for d in incs: if not d.is_dir(): raise FileNotFoundError(f"include_dirs entry is not a directory: {d}") defs = _normalize_defines(defines) xflags = [str(f) for f in extra_flags] lflags = [str(f) for f in link_flags] cache_root = Path(cache_dir).expanduser() if cache_dir else default_cache_dir() root = consumer_root(cache_root, consumer) if cuda_arch is None: cuda_arch = detect_cuda_arch() if cuda_arch == 0: raise RuntimeError("Could not detect the GPU compute capability; pass cuda_arch= explicitly " "(for example cuda_arch=120 for an sm_120 card).") code_options = _code_options( urdf_p, cuda_arch, floating_base=floating_base, ee_joint_names=ee_joint_names, dtype=dtype, algorithm_list=algorithm_list, enable_mujoco_kernels=enable_mujoco_kernels, cost_frames=cost_frames, contact_frames=contact_frames, use_joint_dynamics=use_joint_dynamics, friction_model=friction_model, friction_width=friction_width, runtime_inertia=runtime_inertia, runtime_transform=runtime_transform, runtime_joint_dynamics=runtime_joint_dynamics, runtime_armature=runtime_armature, armature=armature, emit_plant=emit_plant, collision=collision, collision_res=collision_res, collision_native=collision_native, spherized_urdf=spherized_urdf, strict_collision=strict_collision, launch_config_profile=launch_config_profile) identity = artifact_identity(cuda_arch) toolchain = resolve_consumer_toolchain(xflags, lflags) identity["host_cxx"] = toolchain.identity python_abi = _python_abi_identity() if python_extension else None identity["python_abi"] = python_abi input_key = _input_key(urdf_bytes, code_options, srcs, hdrs, incs, defs, xflags, lflags, artifact_name, identity) store = root / "store" def check_inputs_unchanged(): toolchain.environment() current_options = dict(code_options) if collision or collision_native: current_options["collision_meshes"] = _collision_mesh_identity(urdf_p) if spherized_urdf: current_options["spherized_urdf_sha256"] = file_digest(Path(code_options["spherized_urdf"])) current_key = _input_key(urdf_p.read_bytes(), current_options, srcs, hdrs, incs, defs, xflags, lflags, artifact_name, identity) if current_key != input_key: raise RuntimeError("consumer build inputs changed during generation or compilation; " "no artifact published; retry with immutable inputs") def _entry_state(entry_dir: Path) -> list[str]: meta = entry_metadata(entry_dir, artifact_name) if meta is None: return ["entry incomplete (artifact or meta.json missing)"] if not _schema_ok(meta): return ['artifact_schema: incompatible metadata; rebuild artifact'] try: validate_model_association(entry_dir, meta, artifact_name, expected_descriptor_id) except ValueError as exc: return [str(exc)] recorded = read_build_inputs(entry_dir) if recorded is not None and not _schema_ok(recorded): return ['artifact_schema: incompatible identity record; rebuild artifact'] disagreement = _abi_disagreement(meta, recorded, python_extension, python_abi) if disagreement: return [disagreement] stale = stale_hit_reasons(entry_dir, identity) if stale: return stale command = _compile_command(entry_dir, cuda_arch, incs, defs, toolchain) current = discover_dependencies(entry_dir, srcs, command, toolchain) check_inputs_unchanged() if meta.get("dependencies") != current: return ["dependencies: selected files or contents changed (or no capture recorded)"] return [] if plan_only: mapped = keymap_lookup(root, input_key) entry_dir = store / mapped if mapped else None stale = _entry_state(entry_dir) if entry_dir is not None else None return { "consumer": consumer, "name": name, "cache_dir": str(cache_root), "cuda_arch": cuda_arch, "code_options": code_options, "identity": identity, "input_key": input_key, "sources": [str(p) for p in srcs], "headers": [str(p) for p in hdrs], "defines": defs, "content_key": mapped if (mapped and stale == []) else None, "cached": bool(mapped) and stale == [], "stale_reasons": stale, "store_dir": str(entry_dir) if (mapped and stale == []) else None, "would": "load" if (mapped and stale == []) else "generate+compile (the content key decides whether compilation is needed)", } if not force_rebuild: mapped = keymap_lookup(root, input_key) if mapped: entry_dir = store / mapped stale = _entry_state(entry_dir) if not stale: meta = json.loads((entry_dir / "meta.json").read_text()) validate_model_association(entry_dir, meta, artifact_name, expected_descriptor_id) _manifest_register(root, name, mapped, meta) return ConsumerArtifact(consumer, name, entry_dir / artifact_name, entry_dir / "grid.cuh", entry_dir, mapped, input_key, cuda_arch, meta, False) _log.info("grid_rbd: artifact pointer for %s/%s rejected (%s); regenerating", consumer, name, "; ".join(stale)) store.mkdir(parents=True, exist_ok=True) _sweep_dead_staging_dirs(store) gen_dir = staging_dir(store, input_key) compiled = False try: generate_grid_cuh(urdf_p, code_options, gen_dir / "grid.cuh") check_inputs_unchanged() compile_cmd = _compile_command(gen_dir, cuda_arch, incs, defs, toolchain) dependencies = discover_dependencies(gen_dir, srcs, compile_cmd, toolchain) def check_build_inputs(): check_inputs_unchanged() if discover_dependencies(gen_dir, srcs, compile_cmd, toolchain) != dependencies: raise RuntimeError("consumer dependencies changed during build; no artifact published; " "retry with immutable inputs") def check_unit_inputs(unit): # Check the unit being exposed, not every unrelated unit around each # compile. Whole-build checks still bracket final publication. check_inputs_unchanged() current = discover_dependencies(gen_dir, [unit], compile_cmd, toolchain) if current["units"][str(unit)] != dependencies["units"][str(unit)]: raise RuntimeError("consumer dependencies changed during unit compilation; " "retry with immutable inputs") check_build_inputs() content_key = _content_key(gen_dir, srcs, hdrs, compile_cmd, lflags, artifact_name, python_abi=python_abi, toolchain=toolchain.identity, dependencies=dependencies) entry_dir = store / content_key cached_meta = entry_metadata(entry_dir, artifact_name) recorded = read_build_inputs(entry_dir) if cached_meta is not None else None # A sidecar can be damaged independently of the content-addressed binary. # Do not return a hit that load_artifact would reject, or relabel an # incompatible declaration. Rebuild and replace under the publication lock. replace = force_rebuild if cached_meta is not None: replace = replace or _abi_disagreement(cached_meta, recorded, python_extension, python_abi) is not None replace = replace or cached_meta.get("dependencies") != dependencies replace = replace or not _schema_ok(cached_meta) or (recorded is not None and ( not _schema_ok(recorded) or recorded.get('key_schema') != _KEY_SCHEMA)) try: validate_model_association(entry_dir, cached_meta, artifact_name, expected_descriptor_id) except ValueError: replace = True if cached_meta is not None and not replace: check_build_inputs() meta = cached_meta if read_build_inputs(entry_dir) is None: write_build_inputs(entry_dir, identity) else: _log.info("grid_rbd: building consumer artifact %s/%s (%d unit(s))", consumer, name, len(srcs)) t0 = time.monotonic() record = _build(gen_dir, srcs, hdrs, compile_cmd, lflags, gen_dir / artifact_name, root / "objects", reuse_objects=not force_rebuild, validate_unit=check_unit_inputs, python_abi=python_abi, toolchain=toolchain, dependencies=dependencies) _validate_python_exports(gen_dir / artifact_name, python_extension) check_build_inputs() compiled = record["compiled_units"] > 0 meta = { "consumer": consumer, "name": name, "artifact": artifact_name, "urdf": str(urdf_p), "urdf_sha256": hashlib.sha256(urdf_bytes).hexdigest(), "sources": {p.name: file_digest(p) for p in srcs}, "source_paths": [str(p) for p in srcs], "headers": {p.name: file_digest(p) for p in hdrs}, "include_dirs": [str(d) for d in incs], "defines": defs, "extra_flags": xflags, "link_flags": lflags, "python_extension": python_extension, "python_abi": python_abi, "consumer_toolchain": toolchain.identity, "dependencies": dependencies, "code_options": code_options, "cuda_arch": int(cuda_arch), "grid_rbd": package_version(), "artifact_schema": ARTIFACT_SCHEMA, "build_seconds": round(time.monotonic() - t0, 1), "build": record, "model_association": record_model_association(gen_dir, artifact_name), } archs = effective_cuda_archs(cuda_arch) if archs is not None: meta["cuda_archs"] = archs (gen_dir / "meta.json").write_text(json.dumps(meta, indent=2, sort_keys=True)) write_build_inputs(gen_dir, identity) check_build_inputs() meta = publish_entry(gen_dir, entry_dir, artifact_name, force=replace) check_inputs_unchanged() validate_model_association(entry_dir, meta, artifact_name, expected_descriptor_id) finally: shutil.rmtree(gen_dir, ignore_errors=True) keymap_record(root, input_key, content_key) _manifest_register(root, name, content_key, meta) return ConsumerArtifact(consumer, name, entry_dir / artifact_name, entry_dir / "grid.cuh", entry_dir, content_key, input_key, cuda_arch, meta, compiled)
[docs] def load_artifact(consumer: str, name: str, cache_dir: str | Path | None = None, cuda_arch: int | None = None, *, expected_descriptor_id: str | None = None) -> ConsumerArtifact: """The prepared artifact `name` of `consumer`, without a toolchain: the manifest names it, the entry must be whole and recorded for this device (or one of a fat build's devices) and key schema. Declared Python extensions must match this interpreter's ABI. With no detectable GPU and no `cuda_arch`, the recorded device is trusted.""" _check_name(consumer, "consumer") _check_name(name, "name") check_expected_id(expected_descriptor_id) cache_root = Path(cache_dir).expanduser() if cache_dir else default_cache_dir() root = consumer_root(cache_root, consumer) entry = _load_manifest(root)["artifacts"].get(name) if _manifest_path(root).exists() else None if entry is None: raise ArtifactNotFoundError(f"no artifact {name!r} for consumer {consumer!r} under {cache_root}") entry_dir = root / "store" / entry["content_key"] artifact_name = entry.get("artifact", "artifact.so") meta = entry_metadata(entry_dir, artifact_name) if meta is None: raise ArtifactIncompatibleError(f"artifact {name!r} of {consumer!r} is incomplete at {entry_dir}") recorded = read_build_inputs(entry_dir) if recorded is None: raise ArtifactIncompatibleError(f"artifact {name!r} of {consumer!r} predates the identity record") if cuda_arch is None: cuda_arch = detect_cuda_arch() reasons = [] if "python_abi" not in recorded: reasons.append("python_abi: no extension declaration recorded; rebuild the artifact") else: abi = recorded["python_abi"] if (abi is not None and not isinstance(abi, dict)) or ( meta.get("python_extension") is not (abi is not None)) or ( "python_abi" not in meta or meta["python_abi"] != abi): reasons.append("python_abi: metadata and identity declarations disagree; rebuild the artifact") elif abi is not None and abi != _python_abi_identity(): reasons.append(f"python_abi: recorded {abi!r} != current {_python_abi_identity()!r}") if recorded.get("key_schema") != _KEY_SCHEMA: reasons.append(f"key_schema: recorded {recorded.get('key_schema')!r} != current {_KEY_SCHEMA!r}") if not _schema_ok(recorded): reasons.append(f"artifact_schema: recorded {recorded.get('artifact_schema')!r} != current {ARTIFACT_SCHEMA!r}") if not _schema_ok(meta): reasons.append('artifact_schema: metadata is incompatible; rebuild artifact') if cuda_arch and not _arch_loadable(recorded, int(cuda_arch)): reasons.append(f"cuda_arch: recorded {recorded.get('cuda_arch')!r} (archs {recorded.get('cuda_archs')!r}) != current {cuda_arch!r}") if reasons: raise ArtifactIncompatibleError(f"artifact {name!r} of {consumer!r} cannot be loaded here: " + "; ".join(reasons)) try: validate_model_association(entry_dir, meta, artifact_name, expected_descriptor_id) except ValueError as exc: raise ArtifactIncompatibleError(str(exc)) from exc return ConsumerArtifact(consumer, name, entry_dir / artifact_name, entry_dir / "grid.cuh", entry_dir, entry["content_key"], None, int(recorded.get("cuda_arch", cuda_arch or 0)), meta, False)
[docs] def list_artifacts(consumer: str, cache_dir: str | Path | None = None) -> list[dict[str, Any]]: """Every artifact the consumer's manifest names, with its metadata.""" _check_name(consumer, "consumer") cache_root = Path(cache_dir).expanduser() if cache_dir else default_cache_dir() root = consumer_root(cache_root, consumer) if not _manifest_path(root).exists(): return [] return [{"name": n, **e} for n, e in _load_manifest(root)["artifacts"].items()]