diff --git a/.gitignore b/.gitignore index fb8679fa7..d7d7bad33 100644 --- a/.gitignore +++ b/.gitignore @@ -224,3 +224,6 @@ rfc.md # Markdown/mermaid lint tooling deps scripts/lint-mermaid/node_modules/ + +# Nix +result* diff --git a/crates/openshell-bootstrap/Cargo.toml b/crates/openshell-bootstrap/Cargo.toml index 578d59e65..2e53d1d41 100644 --- a/crates/openshell-bootstrap/Cargo.toml +++ b/crates/openshell-bootstrap/Cargo.toml @@ -9,6 +9,10 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[lib] +name = "openshell_bootstrap" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } bollard = "0.20" diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index b03fb1494..8babd7d52 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_core" +path = "src/lib.rs" + [dependencies] prost = { workspace = true } prost-types = { workspace = true } diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 12e79a1dc..0d382d4c1 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -12,8 +12,12 @@ fn main() -> Result<(), Box> { // builds where .git is absent, this silently does nothing and the binary // falls back to CARGO_PKG_VERSION (which is already sed-patched by the // build pipeline). - println!("cargo:rerun-if-changed=../../.git/HEAD"); - println!("cargo:rerun-if-changed=../../.git/refs/tags"); + if Path::new("../../.git/HEAD").exists() { + println!("cargo:rerun-if-changed=../../.git/HEAD"); + } + if Path::new("../../.git/refs/tags").exists() { + println!("cargo:rerun-if-changed=../../.git/refs/tags"); + } if let Some(version) = git_version() { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); @@ -22,15 +26,16 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); - // Use bundled protoc from protobuf-src. The system protoc (from apt-get) - // does not bundle the well-known type includes (google/protobuf/struct.proto - // etc.), so we must use protobuf-src which ships both the binary and the - // include tree. - // SAFETY: This is run at build time in a single-threaded build script context. - // No other threads are reading environment variables concurrently. - #[allow(unsafe_code)] - unsafe { - env::set_var("PROTOC", protobuf_src::protoc()); + if env::var_os("PROTOC").is_none() && !path_has_protoc() { + // Keep non-Nix builds working without requiring users to install protoc. + // Nix builds provide protoc explicitly, so they do not rely on this + // vendored fallback. + // SAFETY: This is run at build time in a single-threaded build script context. + // No other threads are reading environment variables concurrently. + #[allow(unsafe_code)] + unsafe { + env::set_var("PROTOC", protobuf_src::protoc()); + } } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); @@ -72,6 +77,16 @@ fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<() Ok(()) } +fn path_has_protoc() -> bool { + let Some(path) = env::var_os("PATH") else { + return false; + }; + + env::split_paths(&path) + .map(|dir| dir.join(format!("protoc{}", env::consts::EXE_SUFFIX))) + .any(|candidate| candidate.is_file()) +} + /// Derive a version string from `git describe --tags`. /// /// Implements the "guess-next-dev" convention used by the release pipeline diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index e2c97532a..0e423aef7 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_docker" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index c222c9c31..83683943a 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_kubernetes" +path = "src/lib.rs" + [[bin]] name = "openshell-driver-kubernetes" path = "src/main.rs" diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 6f2963d92..41d1b7ccb 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_podman" +path = "src/lib.rs" + [[bin]] name = "openshell-driver-podman" path = "src/main.rs" diff --git a/crates/openshell-ocsf/Cargo.toml b/crates/openshell-ocsf/Cargo.toml index 14cc93ba3..fca761bd1 100644 --- a/crates/openshell-ocsf/Cargo.toml +++ b/crates/openshell-ocsf/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_ocsf" +path = "src/lib.rs" + [dependencies] chrono = { version = "0.4", features = ["serde"] } serde = { workspace = true } diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 8936b85be..82c10a4e1 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_policy" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } serde = { workspace = true } diff --git a/crates/openshell-prover/Cargo.toml b/crates/openshell-prover/Cargo.toml index ee815f3a3..749c05379 100644 --- a/crates/openshell-prover/Cargo.toml +++ b/crates/openshell-prover/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_prover" +path = "src/lib.rs" + [features] bundled-z3 = ["z3/bundled"] diff --git a/crates/openshell-providers/Cargo.toml b/crates/openshell-providers/Cargo.toml index e82574d73..c9345f3e0 100644 --- a/crates/openshell-providers/Cargo.toml +++ b/crates/openshell-providers/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_providers" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } serde = { workspace = true } diff --git a/crates/openshell-router/Cargo.toml b/crates/openshell-router/Cargo.toml index e4c3d5ea7..af3fdb78a 100644 --- a/crates/openshell-router/Cargo.toml +++ b/crates/openshell-router/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_router" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } bytes = { workspace = true } diff --git a/crates/openshell-server-macros/Cargo.toml b/crates/openshell-server-macros/Cargo.toml index f929d43a6..fc04db568 100644 --- a/crates/openshell-server-macros/Cargo.toml +++ b/crates/openshell-server-macros/Cargo.toml @@ -10,6 +10,8 @@ license.workspace = true repository.workspace = true [lib] +name = "openshell_server_macros" +path = "src/lib.rs" proc-macro = true [dependencies] diff --git a/crates/openshell-tui/Cargo.toml b/crates/openshell-tui/Cargo.toml index b0ac0c7ca..386b1243f 100644 --- a/crates/openshell-tui/Cargo.toml +++ b/crates/openshell-tui/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_tui" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } openshell-bootstrap = { path = "../openshell-bootstrap" } diff --git a/crates/openshell-vfio/Cargo.toml b/crates/openshell-vfio/Cargo.toml index b6d7cc3cd..7752d4543 100644 --- a/crates/openshell-vfio/Cargo.toml +++ b/crates/openshell-vfio/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_vfio" +path = "src/lib.rs" + [dependencies] serde = { workspace = true } serde_json = { workspace = true } diff --git a/flake.lock b/flake.lock index 7b9881771..8de4f5362 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1780099841, + "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=", + "owner": "ipetkov", + "repo": "crane", + "rev": "0532eb17955225173906d671fb36306bdeb1e2dc", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,6 +51,7 @@ }, "root": { "inputs": { + "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "rust-overlay": "rust-overlay", diff --git a/flake.nix b/flake.nix index 13c4857bc..50c56474b 100644 --- a/flake.nix +++ b/flake.nix @@ -11,6 +11,7 @@ url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; + crane.url = "github:ipetkov/crane"; treefmt-nix = { url = "github:numtide/treefmt-nix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -22,6 +23,7 @@ flake-utils, nixpkgs, rust-overlay, + crane, treefmt-nix, ... }: @@ -32,18 +34,55 @@ inherit system; overlays = [ (import rust-overlay) ]; }; + lib = pkgs.lib; rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + + craneLib = (crane.mkLib pkgs).overrideToolchain (_: rustToolchain); + + crateSpecs = import ./nix/crate.nix { + inherit pkgs; + root = ./.; + }; + + # Crate-by-crate crane helpers (workspace graph, minimal per-crate + # source, buildWorkspaceCrate). See nix/workspace.nix. + workspace = import ./nix/workspace.nix { + inherit lib pkgs craneLib; + root = ./.; + inherit crateSpecs; + }; + inherit (workspace) buildWorkspaceCrate; + + workspaceCrates = lib.mapAttrs (_: buildWorkspaceCrate) crateSpecs; + crates = { + openshell = workspaceCrates.openshell-cli.package; + openshell-gateway = workspaceCrates.openshell-server.package; + openshell-sandbox = workspaceCrates.openshell-sandbox.package; + openshell-driver-vm = workspaceCrates.openshell-driver-vm.package; + openshell-driver-kubernetes = workspaceCrates.openshell-driver-kubernetes.package; + openshell-driver-podman = workspaceCrates.openshell-driver-podman.package; + }; + treefmtEval = treefmt-nix.lib.evalModule pkgs { projectRootFile = "flake.nix"; programs.nixfmt.enable = true; }; in { + packages = crates // { + default = pkgs.symlinkJoin { + name = "openshell-0.0.0"; + paths = lib.attrValues crates; + }; + }; + devShells.default = pkgs.mkShell { packages = with pkgs; [ rustToolchain # Required to find packages pkg-config + # Required for protobuf code generation. + protobuf # Required for bindgen generation. llvmPackages.libclang # system dependency for openshell-prover diff --git a/nix/crate.nix b/nix/crate.nix new file mode 100644 index 000000000..1458e7ec3 --- /dev/null +++ b/nix/crate.nix @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ + pkgs, + root, +}: +{ + # Each crate declares the compile-time assets and build tools it needs. The + # workspace builder collects nativeBuildInputs/buildInputs/env from the + # transitive Cargo closure. + openshell-bootstrap = { + dir = "openshell-bootstrap"; + assets = [ (root + "/proto") ]; + }; + openshell-cli = { + dir = "openshell-cli"; + assets = [ + (root + "/proto") + (root + "/providers") + (root + "/crates/openshell-prover/registry") + ]; + }; + openshell-server = { + dir = "openshell-server"; + assets = [ + (root + "/proto") + (root + "/providers") + (root + "/crates/openshell-prover/registry") + (root + "/crates/openshell-server/migrations") + ]; + }; + openshell-core = { + dir = "openshell-core"; + nativeBuildInputs = [ pkgs.protobuf ]; + assets = [ (root + "/proto") ]; + }; + openshell-driver-docker = { + dir = "openshell-driver-docker"; + assets = [ (root + "/proto") ]; + }; + openshell-sandbox = { + dir = "openshell-sandbox"; + assets = [ + (root + "/proto") + (root + "/crates/openshell-sandbox/data") + (root + "/crates/openshell-sandbox/src/skills") + ]; + }; + openshell-driver-vm = { + dir = "openshell-driver-vm"; + assets = [ + (root + "/proto") + (root + "/crates/openshell-driver-vm/scripts") + ]; + }; + openshell-driver-kubernetes = { + dir = "openshell-driver-kubernetes"; + assets = [ (root + "/proto") ]; + }; + openshell-driver-podman = { + dir = "openshell-driver-podman"; + assets = [ (root + "/proto") ]; + }; + openshell-ocsf = { + dir = "openshell-ocsf"; + assets = [ (root + "/crates/openshell-ocsf/schemas") ]; + }; + openshell-policy = { + dir = "openshell-policy"; + assets = [ (root + "/proto") ]; + }; + openshell-prover = { + dir = "openshell-prover"; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.z3 ]; + env.LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + assets = [ + (root + "/crates/openshell-prover/registry") + (root + "/crates/openshell-prover/testdata") + ]; + }; + openshell-providers = { + dir = "openshell-providers"; + assets = [ + (root + "/proto") + (root + "/providers") + ]; + }; + openshell-router = { + dir = "openshell-router"; + assets = [ (root + "/proto") ]; + }; + openshell-server-macros = { + dir = "openshell-server-macros"; + }; + openshell-tui = { + dir = "openshell-tui"; + assets = [ + (root + "/proto") + (root + "/providers") + ]; + }; + openshell-vfio = { + dir = "openshell-vfio"; + }; +} diff --git a/nix/workspace.nix b/nix/workspace.nix new file mode 100644 index 000000000..4189d5a85 --- /dev/null +++ b/nix/workspace.nix @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Crate-by-crate crane helpers for a Cargo workspace. +# +# Each crate is built from a minimal source, its own code plus that of its +# transitive workspace dependencies, and gets its own dependency cache. Crates +# outside that closure are reduced to their Cargo.toml so cargo can resolve +# the workspace without their source and without any Cargo.toml edits. +# Editing one crate never rebuilds an unrelated crate, and (because crane +# launders the source before building deps) never rebuilds any crate's +# dependency cache. +{ + lib, + pkgs, + craneLib, + # Workspace root: holds the virtual Cargo.toml, Cargo.lock and .cargo/. + root, + # Member directory, relative to root. + crateDir ? "crates", + # Crate metadata keyed by workspace crate directory. + crateSpecs ? { }, + # Version stamped onto every crate derivation. + version ? "0.0.0", +}: +let + cratesRoot = root + "/${crateDir}"; + + # Workspace dependency graph, derived from the Cargo.tomls + crateDirs = lib.attrNames (lib.filterAttrs (_: t: t == "directory") (builtins.readDir cratesRoot)); + + # Direct intra-workspace path-dependencies of a crate, as dir names. + directDeps = + dir: + let + manifest = builtins.fromTOML (builtins.readFile (cratesRoot + "/${dir}/Cargo.toml")); + in + lib.pipe (manifest.dependencies or { }) [ + lib.attrValues + (lib.filter (v: builtins.isAttrs v && v ? path)) + (map (v: baseNameOf v.path)) + (lib.filter (d: builtins.elem d crateDirs)) + ]; + + # Transitive closure of a crate within the workspace: its own dir plus every workspace dep. + closureOf = + dir: + map (e: e.key) ( + builtins.genericClosure { + startSet = [ { key = dir; } ]; + operator = e: map (key: { inherit key; }) (directDeps e.key); + } + ); + + specFor = dir: lib.attrByPath [ dir ] { } crateSpecs; + + closureList = closure: field: lib.concatLists (map (d: (specFor d).${field} or [ ]) closure); + + closureEnv = closure: lib.foldl' lib.recursiveUpdate { } (map (d: (specFor d).env or { }) closure); + + # Every member's Cargo.toml, cargo must see all of them to resolve the + # workspace even for crates whose source we leave out. + allManifests = map (d: cratesRoot + "/${d}/Cargo.toml") crateDirs; + + # Source tree carrying the real sources of the given crate dirs, plus every + # member's Cargo.toml and the given assets. + mkSrc = + { + dirs, + assets ? [ ], + }: + lib.fileset.toSource { + inherit root; + fileset = lib.fileset.unions ( + [ + (root + "/Cargo.toml") + (root + "/Cargo.lock") + (root + "/.cargo") + ] + ++ allManifests + ++ map (d: craneLib.fileset.commonCargoSources (cratesRoot + "/${d}")) dirs + ++ assets + ); + }; + + # Build one workspace crate (pname == dir) in three cached layers. Every layer + # uses the SAME `-p ` selection, so cargo's feature unification is + # identical across them and the compiled artifacts are reusable: + # 1. crates.io deps — buildDepsOnly; immune to first-party code. + # 2. workspace-dep libs — build `-p ` with the crate's OWN source + # stubbed (real path-deps), so its libs compile with + # the crate's real feature set and get cached. + # 3. the crate itself — reuses 1 + 2; only the crate's own code recompiles. + buildWorkspaceCrate = + { + dir, + assets ? [ ], + nativeBuildInputs ? [ ], + buildInputs ? [ ], + env ? { }, + }: + let + closure = closureOf dir; + workspaceDeps = lib.filter (d: d != dir) closure; + effectiveNativeBuildInputs = lib.unique ( + closureList closure "nativeBuildInputs" ++ nativeBuildInputs + ); + effectiveBuildInputs = lib.unique (closureList closure "buildInputs" ++ buildInputs); + effectiveEnv = lib.recursiveUpdate (closureEnv closure) env; + common = { + pname = dir; + inherit version; + nativeBuildInputs = effectiveNativeBuildInputs; + buildInputs = effectiveBuildInputs; + env = effectiveEnv; + strictDeps = true; + # Build only, skip the cargo test/check phase for now. + doCheck = false; + cargoExtraArgs = "--locked -p ${dir}"; + }; + + cratesDeps = craneLib.buildDepsOnly (common // { src = mkSrc { dirs = [ ]; }; }); + + mkWorkspaceLibsSrc = + let + base = mkSrc { + dirs = workspaceDeps; + inherit assets; + }; + dummyCrate = craneLib.mkDummySrc { src = cratesRoot + "/${dir}"; }; + in + pkgs.runCommandLocal "source" { } '' + cp -r ${base} $out + chmod -R u+w $out + rm -rf "$out/${crateDir}/${dir}" + cp -r ${dummyCrate} "$out/${crateDir}/${dir}" + ''; + + workspaceLibs = + if workspaceDeps == [ ] then + cratesDeps + else + craneLib.buildPackage ( + common + // { + pname = "${dir}-workspace-libs"; + src = mkWorkspaceLibsSrc; + cargoArtifacts = cratesDeps; + doInstallCargoArtifacts = true; + postInstall = '' + cargo clean --release -p ${dir} + ''; + } + ); + in + { + package = craneLib.buildPackage ( + common + // { + src = mkSrc { + dirs = closure; + inherit assets; + }; + cargoArtifacts = workspaceLibs; + } + ); + }; +in +{ + inherit buildWorkspaceCrate; +}