Sparse lock files WIP

CLAUDE.md

@@ -1,5 +1,5 @@
 Observe the following rules when contributing to this repository:
 
-* Before committing, run ./maintainers/format.sh to detect/fix any formatting issues.
+* Before committing, run `./maintainers/format.sh` to detect/fix any formatting issues. If you've only touched C++ files, run `run ./maintainers/format.sh clang-format` since it's a lot faster.
 
 * Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits.

maintainers/format.sh

@@ -9,7 +9,14 @@ if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then
   exit 1;
 fi;
 
-while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files; do
+# The argument is either `--until-stable` or the ID of the single
+# hook to run.
+hook=""
+if [ "${1:-}" != "--until-stable" ]; then
+    hook="${1:-}"
+fi
+
+while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files ${hook:+"$hook"}; do
     if [ "${1:-}" != "--until-stable" ]; then
         exit 1
     fi

src/libcmd/flake-schemas.cc

@@ -10,7 +10,7 @@ namespace nix::flake_schemas {
 using namespace eval_cache;
 using namespace flake;
 
-static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state)
+static std::unique_ptr<LockedFlake> getBuiltinDefaultSchemasFlake(EvalState & state)
 {
     auto accessor = make_ref<MemorySourceAccessor>();
 
@@ -49,11 +49,11 @@ ref<EvalCache> call(
 #include "call-flake-schemas.nix.gen.hh"
         ;
 
-    auto lockedDefaultSchemasFlake = defaultSchemasFlake
-                                         ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {})
-                                         : getBuiltinDefaultSchemasFlake(state);
+    std::shared_ptr<LockedFlake> lockedDefaultSchemasFlake =
+        defaultSchemasFlake ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {})
+                            : getBuiltinDefaultSchemasFlake(state);
     auto lockedDefaultSchemasFlakeFingerprint =
-        lockedDefaultSchemasFlake.getFingerprint(*state.store, state.fetchSettings);
+        lockedDefaultSchemasFlake->getFingerprint(*state.store, state.fetchSettings);
 
     std::optional<Fingerprint> fingerprint2;
     if (allowEvalCache && evalSettings.useEvalCache && evalSettings.pureEval && fingerprint
@@ -78,7 +78,7 @@ ref<EvalCache> call(
                 state.parseExprFromString(callFlakeSchemasNix, state.rootPath(CanonPath::root)), *vCallFlakeSchemas);
 
             auto vFlake = state.allocValue();
-            flake::callFlake(state, *lockedFlake, *vFlake);
+            flake::callFlake(state, lockedFlake, *vFlake);
 
             auto vDefaultSchemasFlake = state.allocValue();
             if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas")))

src/libcmd/include/nix/cmd/command.hh

@@ -6,7 +6,6 @@
 #include "nix/cmd/common-eval-args.hh"
 #include "nix/store/path.hh"
 #include "nix/store/store-reference.hh"
-#include "nix/flake/lockfile.hh"
 
 #include <optional>
 

src/libcmd/installable-flake.cc

@@ -310,7 +310,8 @@ ref<flake::LockedFlake> InstallableFlake::getLockedFlake() const
         flake::LockFlags lockFlagsApplyConfig = lockFlags;
         // FIXME why this side effect?
         lockFlagsApplyConfig.applyNixConfig = true;
-        _lockedFlake = make_ref<flake::LockedFlake>(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
+        _lockedFlake =
+            std::shared_ptr<flake::LockedFlake>(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
     }
     // _lockedFlake is now non-null but still just a shared_ptr
     return ref<flake::LockedFlake>(_lockedFlake);
@@ -328,12 +329,10 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
 {
     auto lockedFlake = getLockedFlake();
 
-    if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
-        if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
-            if (lockedNode->isFlake) {
-                debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
-                return std::move(lockedNode->lockedRef);
-            }
+    if (auto nixpkgsInput = lockedFlake->findInput(*state, lockedFlake->resolveFollows(*state, {"nixpkgs"}))) {
+        if (nixpkgsInput->isFlake) {
+            debug("using nixpkgs flake '%s'", nixpkgsInput->lockedRef);
+            return std::move(nixpkgsInput->lockedRef);
         }
     }
 

src/libcmd/installables.cc

@@ -48,19 +48,9 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "recreate-lock-file",
-        .description = R"(
-    Recreate the flake's lock file from scratch.
-
-    > **DEPRECATED**
-    >
-    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
-        )",
+        .description = "Recreate the flake's lock file from scratch.",
         .category = category,
-        .handler = {[&]() {
-            lockFlags.recreateLockFile = true;
-            warn(
-                "'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
-        }},
+        .handler = {&lockFlags.recreateLockFile, true},
     });
 
     addFlag({
@@ -117,7 +107,7 @@ MixFlakeOptions::MixFlakeOptions()
             if (!path)
                 throw UsageError(
                     "--update-input was passed a zero-length input path, which would refer to the flake itself, not an input");
-            lockFlags.inputUpdates.insert(*path);
+            lockFlags.inputUpdates->insert(*path);
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
@@ -175,23 +165,24 @@ MixFlakeOptions::MixFlakeOptions()
         .labels = {"flake-url"},
         .handler = {[&](std::string flakeRef) {
             auto evalState = getEvalState();
-            auto flake = flake::lockFlake(
+            auto lockedFlake = flake::lockFlake(
                 flakeSettings,
                 *evalState,
                 parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()),
                 {.writeLockFile = false});
-            for (auto & [inputName, input] : flake.lockFile.root->inputs) {
-                auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
-                if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
+
+            for (auto & inputName : lockedFlake->getInputNames(*evalState, {})) {
+                if (auto input =
+                        lockedFlake->findInput(*evalState, lockedFlake->resolveFollows(*evalState, {inputName}))) {
                     fetchers::Attrs extraAttrs;
 
-                    if (!input3->lockedRef.subdir.empty()) {
-                        extraAttrs["dir"] = input3->lockedRef.subdir;
+                    if (!input->lockedRef.subdir.empty()) {
+                        extraAttrs["dir"] = input->lockedRef.subdir;
                     }
 
                     overrideRegistry(
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
-                        input3->lockedRef.input,
+                        input->lockedRef.input,
                         extraAttrs);
                 }
             }

src/libcmd/repl.cc

@@ -20,7 +20,6 @@
 #include "nix/store/outputs-query.hh"
 #include "nix/store/globals.hh"
 #include "nix/flake/flake.hh"
-#include "nix/flake/lockfile.hh"
 #include "nix/util/users.hh"
 #include "nix/cmd/editor-for.hh"
 #include "nix/util/finally.hh"

src/libflake-c/nix_api_flake.cc

@@ -185,9 +185,10 @@ nix_locked_flake * nix_flake_lock(
     nix_clear_err(context);
     try {
         eval_state->state.resetFileCache();
-        auto lockedFlake = nix::make_ref<nix::flake::LockedFlake>(nix::flake::lockFlake(
-            *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags));
-        return new nix_locked_flake{lockedFlake};
+        std::shared_ptr<nix::flake::LockedFlake> lockedFlake(
+            nix::flake::lockFlake(
+                *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags));
+        return new nix_locked_flake{nix::ref(lockedFlake)};
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -203,7 +204,7 @@ nix_value * nix_locked_flake_get_output_attrs(
     nix_clear_err(context);
     try {
         auto v = nix_alloc_value(context, evalState);
-        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value);
+        nix::flake::callFlake(evalState->state, lockedFlake->lockedFlake.get_ptr(), *v->value);
         return v;
     }
     NIXC_CATCH_ERRS_NULL

src/libflake/call-flake.nix

@@ -1,79 +1,64 @@
 # This is a helper to callFlake() to lazily fetch flake inputs.
 
-# The contents of the lock file, in JSON format.
-lockFileStr:
+# An external value wrapping the C++ `LockedFlake` object.
+lockedFlake:
 
-# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
-# with sourceInfo.outPath providing an SourceAccessor to a previously
-# fetched tree. This is necessary for possibly unlocked inputs, in
-# particular the root input, but also --override-inputs pointing to
-# unlocked trees.
-overrides:
+# A primop that, given the locked flake and the input attribute path
+# of an input, returns an attribute set mapping the names of its
+# inputs to either null (for a regular input) or the input attribute
+# path of the target of a "follows" input.
+listFlakeInputs:
+
+# A primop that, given the locked flake and the input attribute path
+# of an input, fetches that input and returns an attribute set
+# describing it.
+fetchFlakeInput:
 
 let
-  inherit (builtins) mapAttrs;
-
-  lockFile = builtins.fromJSON lockFileStr;
-
-  # Resolve a input spec into a node name. An input spec is
-  # either a node name, or a 'follows' path from the root
-  # node.
-  resolveInput =
-    inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec;
-
-  # Follow an input attrpath (e.g. ["dwarffs" "nixpkgs"]) from the
-  # root node, returning the final node.
-  getInputByPath =
-    nodeName: path:
-    if path == [ ] then
-      nodeName
-    else
-      getInputByPath
-        # Since this could be a 'follows' input, call resolveInput.
-        (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
-        (builtins.tail path);
-
-  allNodes = mapAttrs (
-    key: node:
+  inherit (builtins) mapAttrs foldl';
+
+  # Construct the input denoted by the input attribute path
+  # `inputAttrPath` (where `[ ]` denotes the top-level flake). This returns `edges`
+  # (mapping each input name of this input to the input it denotes,
+  # following "follows" indirections) and `result` (the value of this
+  # input, i.e. what ends up in the `inputs` attribute of a flake).
+  mkInput =
+    inputAttrPath:
     let
-      hasOverride = overrides ? ${key};
-      isRelative = node.locked.type or null == "path" && builtins.substring 0 1 node.locked.path != "/";
-
-      parentNode = allNodes.${getInputByPath lockFile.root node.parent};
+      info = fetchFlakeInput lockedFlake inputAttrPath;
 
       sourceInfo =
-        if node.buildTime or false then
+        if info.buildTime then
           derivation {
             name = "source";
             builder = "builtin:fetch-tree";
             system = "builtin";
             __structuredAttrs = true;
-            input = node.locked;
+            input = info.locked;
             outputHashMode = "recursive";
-            outputHash = node.locked.narHash;
+            outputHash = info.locked.narHash;
           }
-        else if hasOverride then
-          overrides.${key}.sourceInfo
-        else if isRelative then
-          parentNode.sourceInfo
         else
-          # FIXME: remove obsolete node.info.
-          # Note: lock file entries are always final.
-          builtins.fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]);
+          info.sourceInfo;
 
-      subdir = overrides.${key}.dir or node.locked.dir or "";
+      subdir = if info.buildTime then info.locked.dir or "" else info.dir;
 
-      outPath =
-        if !hasOverride && isRelative then
-          parentNode.outPath + (if node.locked.path == "" then "" else "/" + node.locked.path)
-        else
-          sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir);
+      outPath = sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir);
 
       flake = import (outPath + "/flake.nix");
 
-      inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) (
-        node.inputs or { }
-      );
+      # Note: constructing `edges` only consults the lock data (via
+      # `listFlakeInputs`), so it never causes anything to be
+      # fetched. A regular input is constructed in place; a "follows"
+      # input is resolved by walking the edges from the top-level
+      # flake, so every distinct input is constructed (and evaluated)
+      # only once.
+      edges = mapAttrs (
+        name: target:
+        if target == null then mkInput (inputAttrPath ++ [ name ]) else getInputByAttrPath target
+      ) (listFlakeInputs lockedFlake inputAttrPath);
+
+      inputs = mapAttrs (name: input: input.result) edges;
 
       outputs = flake.outputs (inputs // { self = result; });
 
@@ -97,17 +82,22 @@ let
 
     in
     {
+      inherit edges;
+
       result =
-        if node.flake or true then
+        if info.flake then
           assert builtins.isFunction flake.outputs;
-          assert !(node.buildTime or false);
+          assert !info.buildTime;
           result
         else
           sourceInfo // { inherit sourceInfo outPath; };
+    };
+
+  # Follow an input attribute path (e.g. ["dwarffs" "nixpkgs"]) from
+  # the top-level flake, returning the final input.
+  getInputByAttrPath = inputAttrPath: foldl' (input: name: input.edges.${name}) root inputAttrPath;
 
-      inherit outPath sourceInfo;
-    }
-  ) lockFile.nodes;
+  root = mkInput [ ];
 
 in
-allNodes.${lockFile.root}.result
+root.result