From 1ae081ebfda6b9091c195b8352f58fdcd326074e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Jul 2026 19:50:14 +0200 Subject: [PATCH 01/26] Rename LockFile to LockFileV7 and move it to lockfile-v7.hh In preparation for the version 8 sparse lock file format, rename LockFile to LockFileV7 and move it, together with the Node and LockedNode data types, to a new header lockfile-v7.hh (implementation in lockfile-v7.cc). lockfile.hh continues to contain the common types like InputAttrPath and NonEmptyInputAttrPath. No behaviour change. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 12 +- src/libflake/include/nix/flake/flake.hh | 4 +- src/libflake/include/nix/flake/lockfile-v7.hh | 101 +++++ src/libflake/include/nix/flake/lockfile.hh | 90 ---- src/libflake/include/nix/flake/meson.build | 1 + src/libflake/lockfile-v7.cc | 399 +++++++++++++++++ src/libflake/lockfile.cc | 400 +----------------- src/libflake/meson.build | 1 + 8 files changed, 516 insertions(+), 492 deletions(-) create mode 100644 src/libflake/include/nix/flake/lockfile-v7.hh create mode 100644 src/libflake/lockfile-v7.cc diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 0ba33ac326f0..5a518b587d8d 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -25,7 +25,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-cache.hh" #include "nix/expr/eval-settings.hh" -#include "nix/flake/lockfile.hh" +#include "nix/flake/lockfile-v7.hh" #include "nix/expr/eval-inline.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" @@ -420,10 +420,10 @@ Flake getFlake( return getFlake(state, originalRef, useRegistries, {}, requireLockable); } -static LockFile readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) +static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { - return lockFilePath.pathExists() ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) - : LockFile(); + return lockFilePath.pathExists() ? LockFileV7(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) + : LockFileV7(); } LockedFlake lockFlake( @@ -475,7 +475,7 @@ LockedFlake lockFlake( explicitCliOverrides.insert(i.first); } - LockFile newLockFile; + LockFileV7 newLockFile; std::vector parents; @@ -850,7 +850,7 @@ LockedFlake lockFlake( /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { - auto diff = LockFile::diff(oldLockFile, newLockFile); + auto diff = LockFileV7::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 99720c9403a0..aeeef9751306 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -3,7 +3,7 @@ #include "nix/util/types.hh" #include "nix/flake/flakeref.hh" -#include "nix/flake/lockfile.hh" +#include "nix/flake/lockfile-v7.hh" #include "nix/expr/value.hh" #include "nix/expr/eval-cache.hh" @@ -139,7 +139,7 @@ typedef Hash Fingerprint; struct LockedFlake { Flake flake; - LockFile lockFile; + LockFileV7 lockFile; /** * Source tree accessors for nodes that have been fetched in diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh new file mode 100644 index 000000000000..555142f51384 --- /dev/null +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -0,0 +1,101 @@ +#pragma once +///@file + +#include "nix/flake/lockfile.hh" + +#include + +namespace nix { +class Store; +class StorePath; +} // namespace nix + +namespace nix::flake { + +struct LockedNode; + +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ +struct Node : std::enable_shared_from_this +{ + typedef std::variant, InputAttrPath> Edge; + + std::map inputs; + + virtual ~Node() {} +}; + +/** + * A non-root node in the lock file. + */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + bool buildTime = false; + + /* The node relative to which relative source paths + (e.g. 'path:../foo') are interpreted. */ + std::optional parentInputAttrPath; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true, + bool buildTime = false, + std::optional parentInputAttrPath = {}) + : lockedRef(std::move(lockedRef)) + , originalRef(std::move(originalRef)) + , isFlake(isFlake) + , buildTime(buildTime) + , parentInputAttrPath(std::move(parentInputAttrPath)) + { + } + + LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); + + StorePath computeStorePath(Store & store) const; +}; + +/** + * The old graph-based lock file format (versions 5-7). + */ +struct LockFileV7 +{ + ref root = make_ref(); + + LockFileV7() {}; + LockFileV7(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path); + + typedef std::map, std::string> KeyMap; + + std::pair toJSON() const; + + std::pair to_string() const; + + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; + + bool operator==(const LockFileV7 & other) const; + + std::shared_ptr findInput(const InputAttrPath & path); + + std::map getAllInputs() const; + + static std::string diff(const LockFileV7 & oldLocks, const LockFileV7 & newLocks); + + /** + * Check that every 'follows' input target exists. + */ + void check(); +}; + +std::ostream & operator<<(std::ostream & stream, const LockFileV7 & lockFile); + +} // namespace nix::flake diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index 27232b20a669..8a686dfbea26 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -3,13 +3,6 @@ #include "nix/flake/flakeref.hh" -#include - -namespace nix { -class Store; -class StorePath; -} // namespace nix - namespace nix::flake { typedef std::vector InputAttrPath; @@ -88,89 +81,6 @@ public: auto operator<=>(const NonEmptyInputAttrPath & other) const = default; }; -struct LockedNode; - -/** - * A node in the lock file. It has outgoing edges to other nodes (its - * inputs). Only the root node has this type; all other nodes have - * type LockedNode. - */ -struct Node : std::enable_shared_from_this -{ - typedef std::variant, InputAttrPath> Edge; - - std::map inputs; - - virtual ~Node() {} -}; - -/** - * A non-root node in the lock file. - */ -struct LockedNode : Node -{ - FlakeRef lockedRef, originalRef; - bool isFlake = true; - bool buildTime = false; - - /* The node relative to which relative source paths - (e.g. 'path:../foo') are interpreted. */ - std::optional parentInputAttrPath; - - LockedNode( - const FlakeRef & lockedRef, - const FlakeRef & originalRef, - bool isFlake = true, - bool buildTime = false, - std::optional parentInputAttrPath = {}) - : lockedRef(std::move(lockedRef)) - , originalRef(std::move(originalRef)) - , isFlake(isFlake) - , buildTime(buildTime) - , parentInputAttrPath(std::move(parentInputAttrPath)) - { - } - - LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); - - StorePath computeStorePath(Store & store) const; -}; - -struct LockFile -{ - ref root = make_ref(); - - LockFile() {}; - LockFile(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path); - - typedef std::map, std::string> KeyMap; - - std::pair toJSON() const; - - std::pair to_string() const; - - /** - * Check whether this lock file has any unlocked or non-final - * inputs. If so, return one. - */ - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; - - bool operator==(const LockFile & other) const; - - std::shared_ptr findInput(const InputAttrPath & path); - - std::map getAllInputs() const; - - static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - - /** - * Check that every 'follows' input target exists. - */ - void check(); -}; - -std::ostream & operator<<(std::ostream & stream, const LockFile & lockFile); - InputAttrPath parseInputAttrPath(std::string_view s); std::string printInputAttrPath(const InputAttrPath & path); diff --git a/src/libflake/include/nix/flake/meson.build b/src/libflake/include/nix/flake/meson.build index fbe54f41208b..081af7f4723c 100644 --- a/src/libflake/include/nix/flake/meson.build +++ b/src/libflake/include/nix/flake/meson.build @@ -5,6 +5,7 @@ include_dirs = [ include_directories('../..') ] headers = files( 'flake.hh', 'flakeref.hh', + 'lockfile-v7.hh', 'lockfile.hh', 'provenance.hh', 'settings.hh', diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc new file mode 100644 index 000000000000..d9230966dccf --- /dev/null +++ b/src/libflake/lockfile-v7.cc @@ -0,0 +1,399 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nix/fetchers/fetch-settings.hh" +#include "nix/flake/lockfile-v7.hh" +#include "nix/util/strings.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/flake/flakeref.hh" +#include "nix/store/path.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/error.hh" +#include "nix/util/fmt.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/logging.hh" +#include "nix/util/ref.hh" +#include "nix/util/types.hh" +#include "nix/util/util.hh" + +namespace nix { +class Store; +} // namespace nix + +namespace nix::flake { + +static FlakeRef +getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = fetchers::jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : fetchers::jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(fetchSettings, attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + +LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) + , parentInputAttrPath( + json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) +{ + if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { + if (lockedRef.input.getNarHash()) + warn( + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", + lockedRef.to_string()); + else + throw Error( + "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); + } + + // For backward compatibility, lock file entries are implicitly final. + assert(!lockedRef.input.attrs.contains("__final")); + lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); +} + +StorePath LockedNode::computeStorePath(Store & store) const +{ + return lockedRef.input.computeStorePath(store); +} + +static std::shared_ptr +doFind(const ref & root, const InputAttrPath & path, std::vector & visited) +{ + auto pos = root; + + auto found = std::find(visited.cbegin(), visited.cend(), path); + + if (found != visited.end()) { + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputAttrPath); + cycle.push_back(printInputAttrPath(path)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(path); + + for (auto & elem : path) { + if (auto i = get(pos->inputs, elem)) { + if (auto node = std::get_if<0>(&*i)) + pos = *node; + else if (auto follows = std::get_if<1>(&*i)) { + if (auto p = doFind(root, *follows, visited)) + pos = ref(p); + else + return {}; + } + } else + return {}; + } + + return pos; +} + +std::shared_ptr LockFileV7::findInput(const InputAttrPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + +LockFileV7::LockFileV7(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path) +{ + auto json = [=] { + try { + return nlohmann::json::parse(contents); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", path, e.what()); + } + }(); + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + std::string rootKey = json["root"]; + std::map> nodeMap{{rootKey, root}}; + + [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { + if (jsonNode.find("inputs") == jsonNode.end()) + return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { // FIXME: remove, obsolete + InputAttrPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto & nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = make_ref(fetchSettings, *jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, *jsonNode2); + } + if (auto child = k->second.dynamic_pointer_cast()) + node.inputs.insert_or_assign(i.key(), ref(child)); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); + } + } + }(*root, json["nodes"][rootKey]); + + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. +} + +std::pair LockFileV7::toJSON() const +{ + nlohmann::json nodes; + KeyMap nodeKeys; + boost::unordered_flat_set keys; + + auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2;; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } + } + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = node.dynamic_pointer_cast()) { + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); + assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); + if (!lockedNode->isFlake) + n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; + if (lockedNode->parentInputAttrPath) + n["parent"] = *lockedNode->parentInputAttrPath; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return {json, std::move(nodeKeys)}; +} + +std::pair LockFileV7::to_string() const +{ + auto [json, nodeKeys] = toJSON(); + return {json.dump(2), std::move(nodeKeys)}; +} + +std::ostream & operator<<(std::ostream & stream, const LockFileV7 & lockFile) +{ + stream << lockFile.toJSON().first.dump(2); + return stream; +} + +std::optional LockFileV7::isUnlocked(const fetchers::Settings & fetchSettings) const +{ + std::set> nodes; + + [&](this const auto & visit, ref node) { + if (!nodes.insert(node).second) + return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }(root); + + /* Return whether the input is either locked, or, if + `allow-dirty-locks` is enabled, it has a NAR hash. In the + latter case, we can verify the input but we may not be able to + fetch it from anywhere. */ + auto isConsideredLocked = [&](const fetchers::Input & input) { + return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); + }; + + for (auto & i : nodes) { + if (i == ref(root)) + continue; + auto node = i.dynamic_pointer_cast(); + if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) + && !node->lockedRef.input.isRelative()) + return node->lockedRef; + } + + return {}; +} + +bool LockFileV7::operator==(const LockFileV7 & other) const +{ + // FIXME: slow + return toJSON().first == other.toJSON().first; +} + +std::map LockFileV7::getAllInputs() const +{ + std::set> done; + std::map res; + + [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { + if (!done.insert(node).second) + return; + + for (auto & [id, input] : node->inputs) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + res.emplace(inputAttrPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputAttrPath, *child); + } + }({}, root); + + return res; +} + +static std::string describe(const FlakeRef & flakeRef) +{ + auto s = fmt("'%s'", flakeRef.to_string(true)); + + if (auto lastModified = flakeRef.input.getLastModified()) + s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); + + return s; +} + +std::ostream & operator<<(std::ostream & stream, const Node::Edge & edge) +{ + if (auto node = std::get_if<0>(&edge)) + stream << describe((*node)->lockedRef); + else if (auto follows = std::get_if<1>(&edge)) + stream << fmt("follows '%s'", printInputAttrPath(*follows)); + return stream; +} + +static bool equals(const Node::Edge & e1, const Node::Edge & e2) +{ + if (auto n1 = std::get_if<0>(&e1)) + if (auto n2 = std::get_if<0>(&e2)) + return (*n1)->lockedRef == (*n2)->lockedRef; + if (auto f1 = std::get_if<1>(&e1)) + if (auto f2 = std::get_if<1>(&e2)) + return *f1 == *f2; + return false; +} + +std::string LockFileV7::diff(const LockFileV7 & oldLocks, const LockFileV7 & newLocks) +{ + auto oldFlat = oldLocks.getAllInputs(); + auto newFlat = newLocks.getAllInputs(); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += fmt( + "• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", printInputAttrPath(j->first), j->second); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); + ++i; + } else { + if (!equals(i->second, j->second)) { + res += + fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", + printInputAttrPath(i->first), + i->second, + j->second); + } + ++i; + ++j; + } + } + + return res; +} + +void LockFileV7::check() +{ + auto inputs = getAllInputs(); + + for (auto & [inputAttrPath, input] : inputs) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !findInput(*follows)) + throw Error( + "input '%s' follows a non-existent input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(*follows)); + } + } +} + +} // namespace nix::flake diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index c44ae77dff91..9bde587f9bb6 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -1,303 +1,16 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include #include #include -#include -#include #include -#include "nix/fetchers/fetch-settings.hh" #include "nix/flake/lockfile.hh" -#include "nix/util/strings.hh" -#include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" #include "nix/flake/flakeref.hh" -#include "nix/store/path.hh" -#include "nix/util/ansicolor.hh" #include "nix/util/error.hh" -#include "nix/util/fmt.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/logging.hh" -#include "nix/util/ref.hh" -#include "nix/util/types.hh" -#include "nix/util/util.hh" - -namespace nix { -class Store; -} // namespace nix +#include "nix/util/strings.hh" namespace nix::flake { -static FlakeRef -getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) -{ - auto i = json.find(attr); - if (i != json.end()) { - auto attrs = fetchers::jsonToAttrs(*i); - // FIXME: remove when we drop support for version 5. - if (info) { - auto j = json.find(info); - if (j != json.end()) { - for (auto k : fetchers::jsonToAttrs(*j)) - attrs.insert_or_assign(k.first, k.second); - } - } - return FlakeRef::fromAttrs(fetchSettings, attrs); - } - - throw Error("attribute '%s' missing in lock file", attr); -} - -LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) - : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) - , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) - , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) - , parentInputAttrPath( - json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) -{ - if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { - if (lockedRef.input.getNarHash()) - warn( - "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " - "This is not reproducible and will break after garbage collection or when shared.", - lockedRef.to_string()); - else - throw Error( - "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", - fetchers::attrsToJSON(lockedRef.input.toAttrs())); - } - - // For backward compatibility, lock file entries are implicitly final. - assert(!lockedRef.input.attrs.contains("__final")); - lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); -} - -StorePath LockedNode::computeStorePath(Store & store) const -{ - return lockedRef.input.computeStorePath(store); -} - -static std::shared_ptr -doFind(const ref & root, const InputAttrPath & path, std::vector & visited) -{ - auto pos = root; - - auto found = std::find(visited.cbegin(), visited.cend(), path); - - if (found != visited.end()) { - std::vector cycle; - std::transform(found, visited.cend(), std::back_inserter(cycle), printInputAttrPath); - cycle.push_back(printInputAttrPath(path)); - throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); - } - visited.push_back(path); - - for (auto & elem : path) { - if (auto i = get(pos->inputs, elem)) { - if (auto node = std::get_if<0>(&*i)) - pos = *node; - else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = doFind(root, *follows, visited)) - pos = ref(p); - else - return {}; - } - } else - return {}; - } - - return pos; -} - -std::shared_ptr LockFile::findInput(const InputAttrPath & path) -{ - std::vector visited; - return doFind(root, path, visited); -} - -LockFile::LockFile(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path) -{ - auto json = [=] { - try { - return nlohmann::json::parse(contents); - } catch (const nlohmann::json::parse_error & e) { - throw Error("Could not parse '%s': %s", path, e.what()); - } - }(); - auto version = json.value("version", 0); - if (version < 5 || version > 7) - throw Error("lock file '%s' has unsupported version %d", path, version); - - std::string rootKey = json["root"]; - std::map> nodeMap{{rootKey, root}}; - - [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { - if (jsonNode.find("inputs") == jsonNode.end()) - return; - for (auto & i : jsonNode["inputs"].items()) { - if (i.value().is_array()) { // FIXME: remove, obsolete - InputAttrPath path; - for (auto & j : i.value()) - path.push_back(j); - node.inputs.insert_or_assign(i.key(), path); - } else { - std::string inputKey = i.value(); - auto k = nodeMap.find(inputKey); - if (k == nodeMap.end()) { - auto & nodes = json["nodes"]; - auto jsonNode2 = nodes.find(inputKey); - if (jsonNode2 == nodes.end()) - throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(fetchSettings, *jsonNode2); - k = nodeMap.insert_or_assign(inputKey, input).first; - getInputs(*input, *jsonNode2); - } - if (auto child = k->second.dynamic_pointer_cast()) - node.inputs.insert_or_assign(i.key(), ref(child)); - else - // FIXME: replace by follows node - throw Error("lock file contains cycle to root node"); - } - } - }(*root, json["nodes"][rootKey]); - - // FIXME: check that there are no cycles in version >= 7. Cycles - // between inputs are only possible using 'follows' indirections. - // Once we drop support for version <= 6, we can simplify the code - // a bit since we don't need to worry about cycles. -} - -std::pair LockFile::toJSON() const -{ - nlohmann::json nodes; - KeyMap nodeKeys; - boost::unordered_flat_set keys; - - auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { - auto k = nodeKeys.find(node); - if (k != nodeKeys.end()) - return k->second; - - if (!keys.insert(key).second) { - for (int n = 2;; ++n) { - auto k = fmt("%s_%d", key, n); - if (keys.insert(k).second) { - key = k; - break; - } - } - } - - nodeKeys.insert_or_assign(node, key); - - auto n = nlohmann::json::object(); - - if (!node->inputs.empty()) { - auto inputs = nlohmann::json::object(); - for (auto & i : node->inputs) { - if (auto child = std::get_if<0>(&i.second)) { - inputs[i.first] = dumpNode(i.first, *child); - } else if (auto follows = std::get_if<1>(&i.second)) { - auto arr = nlohmann::json::array(); - for (auto & x : *follows) - arr.push_back(x); - inputs[i.first] = std::move(arr); - } - } - n["inputs"] = std::move(inputs); - } - - if (auto lockedNode = node.dynamic_pointer_cast()) { - n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); - n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); - if (!lockedNode->isFlake) - n["flake"] = false; - if (lockedNode->buildTime) - n["buildTime"] = true; - if (lockedNode->parentInputAttrPath) - n["parent"] = *lockedNode->parentInputAttrPath; - } - - nodes[key] = std::move(n); - - return key; - }; - - nlohmann::json json; - json["version"] = 7; - json["root"] = dumpNode("root", root); - json["nodes"] = std::move(nodes); - - return {json, std::move(nodeKeys)}; -} - -std::pair LockFile::to_string() const -{ - auto [json, nodeKeys] = toJSON(); - return {json.dump(2), std::move(nodeKeys)}; -} - -std::ostream & operator<<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJSON().first.dump(2); - return stream; -} - -std::optional LockFile::isUnlocked(const fetchers::Settings & fetchSettings) const -{ - std::set> nodes; - - [&](this const auto & visit, ref node) { - if (!nodes.insert(node).second) - return; - for (auto & i : node->inputs) - if (auto child = std::get_if<0>(&i.second)) - visit(*child); - }(root); - - /* Return whether the input is either locked, or, if - `allow-dirty-locks` is enabled, it has a NAR hash. In the - latter case, we can verify the input but we may not be able to - fetch it from anywhere. */ - auto isConsideredLocked = [&](const fetchers::Input & input) { - return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); - }; - - for (auto & i : nodes) { - if (i == ref(root)) - continue; - auto node = i.dynamic_pointer_cast(); - if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) - && !node->lockedRef.input.isRelative()) - return node->lockedRef; - } - - return {}; -} - -bool LockFile::operator==(const LockFile & other) const -{ - // FIXME: slow - return toJSON().first == other.toJSON().first; -} - InputAttrPath parseInputAttrPath(std::string_view s) { InputAttrPath path; @@ -311,6 +24,11 @@ InputAttrPath parseInputAttrPath(std::string_view s) return path; } +std::string printInputAttrPath(const InputAttrPath & path) +{ + return concatStringsSep("/", path); +} + std::optional NonEmptyInputAttrPath::parse(std::string_view s) { auto path = parseInputAttrPath(s); @@ -324,110 +42,4 @@ std::optional NonEmptyInputAttrPath::make(InputAttrPath p return NonEmptyInputAttrPath{std::move(path)}; } -std::map LockFile::getAllInputs() const -{ - std::set> done; - std::map res; - - [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { - if (!done.insert(node).second) - return; - - for (auto & [id, input] : node->inputs) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(id); - res.emplace(inputAttrPath, input); - if (auto child = std::get_if<0>(&input)) - recurse(inputAttrPath, *child); - } - }({}, root); - - return res; -} - -static std::string describe(const FlakeRef & flakeRef) -{ - auto s = fmt("'%s'", flakeRef.to_string(true)); - - if (auto lastModified = flakeRef.input.getLastModified()) - s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); - - return s; -} - -std::ostream & operator<<(std::ostream & stream, const Node::Edge & edge) -{ - if (auto node = std::get_if<0>(&edge)) - stream << describe((*node)->lockedRef); - else if (auto follows = std::get_if<1>(&edge)) - stream << fmt("follows '%s'", printInputAttrPath(*follows)); - return stream; -} - -static bool equals(const Node::Edge & e1, const Node::Edge & e2) -{ - if (auto n1 = std::get_if<0>(&e1)) - if (auto n2 = std::get_if<0>(&e2)) - return (*n1)->lockedRef == (*n2)->lockedRef; - if (auto f1 = std::get_if<1>(&e1)) - if (auto f2 = std::get_if<1>(&e2)) - return *f1 == *f2; - return false; -} - -std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) -{ - auto oldFlat = oldLocks.getAllInputs(); - auto newFlat = newLocks.getAllInputs(); - - auto i = oldFlat.begin(); - auto j = newFlat.begin(); - std::string res; - - while (i != oldFlat.end() || j != newFlat.end()) { - if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { - res += fmt( - "• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", printInputAttrPath(j->first), j->second); - ++j; - } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { - res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); - ++i; - } else { - if (!equals(i->second, j->second)) { - res += - fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", - printInputAttrPath(i->first), - i->second, - j->second); - } - ++i; - ++j; - } - } - - return res; -} - -void LockFile::check() -{ - auto inputs = getAllInputs(); - - for (auto & [inputAttrPath, input] : inputs) { - if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !findInput(*follows)) - throw Error( - "input '%s' follows a non-existent input '%s'", - printInputAttrPath(inputAttrPath), - printInputAttrPath(*follows)); - } - } -} - -void check(); - -std::string printInputAttrPath(const InputAttrPath & path) -{ - return concatStringsSep("/", path); -} - } // namespace nix::flake diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 1bf6f7e5fba6..9f326f9b20a2 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -44,6 +44,7 @@ sources = files( 'flake-primops.cc', 'flake.cc', 'flakeref.cc', + 'lockfile-v7.cc', 'lockfile.cc', 'provenance.cc', 'settings.cc', From a23b4ccdf22b07cc2d023d4b21bbc1a7bf39db9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Jul 2026 19:54:44 +0200 Subject: [PATCH 02/26] maintainers/format.sh: Allow running just one hook --- CLAUDE.md | 2 +- maintainers/format.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 27268bf11f5f..6c5caf64af6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,5 @@ Observe the following rules when contributing to this repository: -* Before committing, run ./maintainers/format.sh to detect/fix any formatting issues. +* Before committing, run `./maintainers/format.sh` to detect/fix any formatting issues. If you've only touched C++ files, run `run ./maintainers/format.sh clang-format` since it's a lot faster. * Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits. diff --git a/maintainers/format.sh b/maintainers/format.sh index b2902e6dc6c8..c15d47a7faab 100755 --- a/maintainers/format.sh +++ b/maintainers/format.sh @@ -9,7 +9,14 @@ if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then exit 1; fi; -while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files; do +# The argument is either `--until-stable` or the ID of the single +# hook to run. +hook="" +if [ "${1:-}" != "--until-stable" ]; then + hook="${1:-}" +fi + +while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files ${hook:+"$hook"}; do if [ "${1:-}" != "--until-stable" ]; then exit 1 fi From 938e69b0aba6bb69bb718ee0ccd54889fbfed2fc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2026 12:06:06 +0200 Subject: [PATCH 03/26] Rename lockfile.{cc,hh} -> input-attr-path.{cc,hh} --- src/libcmd/include/nix/cmd/command.hh | 1 - src/libcmd/repl.cc | 1 - src/libflake/include/nix/flake/flake.hh | 1 + .../include/nix/flake/{lockfile.hh => input-attr-path.hh} | 0 src/libflake/include/nix/flake/lockfile-v7.hh | 2 +- src/libflake/include/nix/flake/meson.build | 2 +- src/libflake/{lockfile.cc => input-attr-path.cc} | 2 +- src/libflake/meson.build | 2 +- 8 files changed, 5 insertions(+), 6 deletions(-) rename src/libflake/include/nix/flake/{lockfile.hh => input-attr-path.hh} (100%) rename src/libflake/{lockfile.cc => input-attr-path.cc} (96%) diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index c43b4f87ed43..2d47a9d82248 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -6,7 +6,6 @@ #include "nix/cmd/common-eval-args.hh" #include "nix/store/path.hh" #include "nix/store/store-reference.hh" -#include "nix/flake/lockfile.hh" #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 6eaab847a0a6..a14a6b2cf813 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -20,7 +20,6 @@ #include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" #include "nix/flake/flake.hh" -#include "nix/flake/lockfile.hh" #include "nix/util/users.hh" #include "nix/cmd/editor-for.hh" #include "nix/util/finally.hh" diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index aeeef9751306..44d545344eee 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -3,6 +3,7 @@ #include "nix/util/types.hh" #include "nix/flake/flakeref.hh" +#include "nix/flake/input-attr-path.hh" #include "nix/flake/lockfile-v7.hh" #include "nix/expr/value.hh" #include "nix/expr/eval-cache.hh" diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/input-attr-path.hh similarity index 100% rename from src/libflake/include/nix/flake/lockfile.hh rename to src/libflake/include/nix/flake/input-attr-path.hh diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index 555142f51384..55bc30482991 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/flake/lockfile.hh" +#include "nix/flake/input-attr-path.hh" #include diff --git a/src/libflake/include/nix/flake/meson.build b/src/libflake/include/nix/flake/meson.build index 081af7f4723c..404ee22a3cf5 100644 --- a/src/libflake/include/nix/flake/meson.build +++ b/src/libflake/include/nix/flake/meson.build @@ -5,8 +5,8 @@ include_dirs = [ include_directories('../..') ] headers = files( 'flake.hh', 'flakeref.hh', + 'input-attr-path.hh', 'lockfile-v7.hh', - 'lockfile.hh', 'provenance.hh', 'settings.hh', 'url-name.hh', diff --git a/src/libflake/lockfile.cc b/src/libflake/input-attr-path.cc similarity index 96% rename from src/libflake/lockfile.cc rename to src/libflake/input-attr-path.cc index 9bde587f9bb6..46ee6e720454 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/input-attr-path.cc @@ -4,7 +4,7 @@ #include #include -#include "nix/flake/lockfile.hh" +#include "nix/flake/input-attr-path.hh" #include "nix/flake/flakeref.hh" #include "nix/util/error.hh" #include "nix/util/strings.hh" diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 9f326f9b20a2..33de5958c8c8 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -44,8 +44,8 @@ sources = files( 'flake-primops.cc', 'flake.cc', 'flakeref.cc', + 'input-attr-path.cc', 'lockfile-v7.cc', - 'lockfile.cc', 'provenance.cc', 'settings.cc', 'url-name.cc', From 2c02256654c10372c5a8469119a7021db39a1057 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2026 13:56:48 +0200 Subject: [PATCH 04/26] Start making LockedFlake an abstract class In preparation for the version 8 sparse lock file format, turn `LockedFlake` into an abstract base class that doesn't expose the version 7 lock file representation, and move that representation into a new subclass `LockedFlakeV7`. The `lockFlake()` functions now return a `std::unique_ptr`. Code that needs to query a locked flake now goes through virtual methods instead of inspecting the version 7 node graph directly: * `isUnlocked()`: check for unlocked or non-final inputs. This allows `getFingerprint()` to be implemented entirely in terms of the abstract interface. * `toJSON()`, with non-virtual `to_string()` and `operator <<` on top of it. * `getInputNames()`: return the names of the inputs of the input denoted by an attribute path prefix. * `findInput()`: look up an input by attribute path, resolving "follows" indirections, and return an `InputInfo` struct containing its locked reference. Used by `--inputs-from` and `InstallableFlake::nixpkgsFlakeRef()`. * `visit()`: walk all transitive inputs in depth-first order, calling a callback with either an `InputInfo` or the target path of a "follows" input. The callback controls recursion, e.g. to skip build-time inputs. Used by `nix flake metadata`, `nix flake archive` and `nix flake prefetch-inputs`. The only remaining uses of `LockedFlakeV7` are in `lockFlake()` and `callFlake()` in libflake itself, which will become the dispatch points between the version 7 and version 8 implementations. Assisted-by: Claude Fable 5 --- src/libcmd/flake-schemas.cc | 12 +- src/libcmd/installable-flake.cc | 15 +- src/libcmd/installables.cc | 15 +- src/libcmd/repl.cc | 2 +- src/libflake-c/nix_api_flake.cc | 7 +- src/libflake/flake-primops.cc | 6 +- src/libflake/flake.cc | 35 +++-- src/libflake/include/nix/flake/flake.hh | 76 ++++++++-- src/libflake/include/nix/flake/lockfile-v7.hh | 34 ++++- src/libflake/lockfile-v7.cc | 62 ++++++++- src/nix/flake-command.hh | 2 +- src/nix/flake-prefetch-inputs.cc | 53 ++++--- src/nix/flake.cc | 130 ++++++++++-------- 13 files changed, 302 insertions(+), 147 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index c8e2992be7e2..9624a1c9ebe5 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -10,7 +10,7 @@ namespace nix::flake_schemas { using namespace eval_cache; using namespace flake; -static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state) +static std::unique_ptr getBuiltinDefaultSchemasFlake(EvalState & state) { auto accessor = make_ref(); @@ -49,11 +49,11 @@ ref call( #include "call-flake-schemas.nix.gen.hh" ; - auto lockedDefaultSchemasFlake = defaultSchemasFlake - ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {}) - : getBuiltinDefaultSchemasFlake(state); + std::shared_ptr lockedDefaultSchemasFlake = + defaultSchemasFlake ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {}) + : getBuiltinDefaultSchemasFlake(state); auto lockedDefaultSchemasFlakeFingerprint = - lockedDefaultSchemasFlake.getFingerprint(*state.store, state.fetchSettings); + lockedDefaultSchemasFlake->getFingerprint(*state.store, state.fetchSettings); std::optional fingerprint2; if (allowEvalCache && evalSettings.useEvalCache && evalSettings.pureEval && fingerprint @@ -84,7 +84,7 @@ ref call( if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas"))) vDefaultSchemasFlake->mkNull(); else - flake::callFlake(state, lockedDefaultSchemasFlake, *vDefaultSchemasFlake); + flake::callFlake(state, *lockedDefaultSchemasFlake, *vDefaultSchemasFlake); auto vRes = state.allocValue(); Value * args[] = {vDefaultSchemasFlake, vFlake}; diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d2022aeb09d0..778c013980cb 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -310,7 +310,8 @@ ref InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = make_ref(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = + std::shared_ptr(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } // _lockedFlake is now non-null but still just a shared_ptr return ref(_lockedFlake); @@ -326,14 +327,10 @@ ref InstallableFlake::openEvalCache() const FlakeRef InstallableFlake::nixpkgsFlakeRef() const { - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { - if (lockedNode->isFlake) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } + if (auto nixpkgsInput = getLockedFlake()->findInput({"nixpkgs"})) { + if (nixpkgsInput->isFlake) { + debug("using nixpkgs flake '%s'", nixpkgsInput->lockedRef); + return std::move(nixpkgsInput->lockedRef); } } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2285e0c3cf51..3cc343faccf2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -175,23 +175,24 @@ MixFlakeOptions::MixFlakeOptions() .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); - auto flake = flake::lockFlake( + auto lockedFlake = flake::lockFlake( flakeSettings, *evalState, parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()), {.writeLockFile = false}); - for (auto & [inputName, input] : flake.lockFile.root->inputs) { - auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes - if (auto input3 = std::dynamic_pointer_cast(input2)) { + + for (auto & inputName : lockedFlake->getInputNames({})) { + // Note: findInput() resolves 'follows' nodes. + if (auto input = lockedFlake->findInput({inputName})) { fetchers::Attrs extraAttrs; - if (!input3->lockedRef.subdir.empty()) { - extraAttrs["dir"] = input3->lockedRef.subdir; + if (!input->lockedRef.subdir.empty()) { + extraAttrs["dir"] = input->lockedRef.subdir; } overrideRegistry( fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}), - input3->lockedRef.input, + input->lockedRef.input, extraAttrs); } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index a14a6b2cf813..20478296b43e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -730,7 +730,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) flake::callFlake( *state, - flake::lockFlake( + *flake::lockFlake( flakeSettings, *state, flakeRef, diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 2558236a7e5a..6c474999f31c 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -185,9 +185,10 @@ nix_locked_flake * nix_flake_lock( nix_clear_err(context); try { eval_state->state.resetFileCache(); - auto lockedFlake = nix::make_ref(nix::flake::lockFlake( - *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); - return new nix_locked_flake{lockedFlake}; + std::shared_ptr lockedFlake( + nix::flake::lockFlake( + *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); + return new nix_locked_flake{nix::ref(lockedFlake)}; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index b8838ab18e52..1e8fea69f67f 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -42,7 +42,7 @@ PrimOp getFlake(const Settings & settings) if (args[0]->type() == nPath) { auto path = state.realisePath(pos, *args[0]); - callFlake(state, lockFlake(settings, state, path, lockFlags), v); + callFlake(state, *lockFlake(settings, state, path, lockFlags), v); } else { NixStringContext context; std::string flakeRefS( @@ -73,12 +73,12 @@ PrimOp getFlake(const Settings & settings) auto path = state.storePath(storePath) / CanonPath(subPath); if (!flakeRef.subdir.empty()) path = path / flakeRef.subdir; - return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + return callFlake(state, *lockFlake(settings, state, path, lockFlags), v); } } } - callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); + callFlake(state, *lockFlake(settings, state, flakeRef, lockFlags), v); } }; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 5a518b587d8d..dce43df6a922 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -426,7 +426,7 @@ static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const S : LockFileV7(); } -LockedFlake lockFlake( +std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); @@ -446,7 +446,7 @@ LockedFlake lockFlake( auto oldLockFile = readLockFile(state.fetchSettings, lockFlags.referenceLockFilePath.value_or(flake.lockFilePath())); - debug("old lock file: %s", oldLockFile); + debug("old lock file: %s", oldLockFile.to_string().first); struct OverrideTarget { @@ -843,7 +843,7 @@ LockedFlake lockFlake( /* Check 'follows' inputs. */ newLockFile.check(); - debug("new lock file: %s", newLockFile); + debug("new lock file: %s", newLockFile.to_string().first); auto sourcePath = topRef.input.getSourcePath(); @@ -872,7 +872,7 @@ LockedFlake lockFlake( "flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto newLockFileS = fmt("%s\n", newLockFile); + auto newLockFileS = fmt("%s\n", newLockFile.to_string().first); if (lockFlags.outputLockFilePath) { if (lockFlags.commitLockFile) @@ -934,8 +934,7 @@ LockedFlake lockFlake( } } - return LockedFlake{ - .flake = std::move(flake), .lockFile = std::move(newLockFile), .nodePaths = std::move(nodePaths)}; + return std::make_unique(std::move(flake), std::move(newLockFile), std::move(nodePaths)); } catch (Error & e) { e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); @@ -943,7 +942,7 @@ LockedFlake lockFlake( } } -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); @@ -952,7 +951,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {}, false)); } -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags) { /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */ @@ -981,8 +980,10 @@ static Value * requireInternalFile(EvalState & state, CanonPath path) return v; } -void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) +void callFlake(EvalState & state, const LockedFlake & _lockedFlake, Value & vRes) { + auto & lockedFlake = dynamic_cast(_lockedFlake); + auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); @@ -1023,16 +1024,28 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) state.callFunction(*vCallFlake, args, vRes, noPos); } +LockedFlake::~LockedFlake() {} + +std::string LockedFlake::to_string() const +{ + return toJSON().dump(2); +} + +std::ostream & operator<<(std::ostream & stream, const LockedFlake & lockedFlake) +{ + return stream << lockedFlake.to_string(); +} + std::optional LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const { - if (lockFile.isUnlocked(fetchSettings)) + if (isUnlocked(fetchSettings)) return std::nullopt; auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; - *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, *this); if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { if (std::get_if(revCount)) { diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 44d545344eee..3b1b0006e8dd 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -4,10 +4,11 @@ #include "nix/util/types.hh" #include "nix/flake/flakeref.hh" #include "nix/flake/input-attr-path.hh" -#include "nix/flake/lockfile-v7.hh" #include "nix/expr/value.hh" #include "nix/expr/eval-cache.hh" +#include + namespace nix { class EvalState; @@ -87,7 +88,8 @@ struct Flake FlakeRef resolvedRef; /** - * the specific local store result of invoking the fetcher + * The flakeref returned by the fetcher. Note that this is a misnomer and it might not actually be locked (e.g. a + * dirty Git repo). */ FlakeRef lockedRef; @@ -140,18 +142,72 @@ typedef Hash Fingerprint; struct LockedFlake { Flake flake; - LockFileV7 lockFile; + + LockedFlake(Flake && flake) + : flake(std::move(flake)) + { + } + + virtual ~LockedFlake(); + + /** + * Return the names of the inputs of the input denoted by + * `prefix`, or of the top-level flake if `prefix` is empty. + */ + virtual std::vector getInputNames(const InputAttrPath & prefix) const = 0; + + /** + * Information about a locked input. + */ + struct InputInfo + { + FlakeRef lockedRef; + bool isFlake = true; + bool buildTime = false; + }; /** - * Source tree accessors for nodes that have been fetched in - * lockFlake(); in particular, the root node and the overridden - * inputs. + * Return information about the input denoted by `path`, resolving + * 'follows' indirections. Returns std::nullopt if the input does + * not exist. */ - std::map, SourcePath> nodePaths; + virtual std::optional findInput(const InputAttrPath & path) const = 0; + + /** + * Callback for `visit()`. The second argument is either an + * `InputInfo` for locked inputs, or, for "follows" inputs, the + * input attribute path of the target of the "follows" (relative + * to the top-level flake). The return value denotes whether + * `visit()` should recurse into the inputs of this input. + */ + using VisitCallback = + std::function & input)>; + + /** + * Call `callback` for every transitive input of this flake, + * including the root (which has the empty input attribute + * path). Inputs are visited in depth-first order, parents before + * children. If the callback returns false, we do not recurse into + * the inputs of that input. We never recurse into "follows" + * inputs; their targets are visited under their own paths. + */ + virtual void visit(VisitCallback callback) const = 0; std::optional getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; + + /** + * Check whether the lock file has any unlocked or non-final + * inputs. If so, return one. + */ + virtual std::optional isUnlocked(const fetchers::Settings & fetchSettings) const = 0; + + virtual nlohmann::json toJSON() const = 0; + + std::string to_string() const; }; +std::ostream & operator<<(std::ostream & stream, const LockedFlake & lockedFlake); + struct LockFlags { /** @@ -249,13 +305,13 @@ Flake readFlake( * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is * writable. */ -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); -LockedFlake lockFlake( +std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake); -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index 55bc30482991..ed67903a4175 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/flake/input-attr-path.hh" +#include "nix/flake/flake.hh" #include @@ -84,7 +84,7 @@ struct LockFileV7 bool operator==(const LockFileV7 & other) const; - std::shared_ptr findInput(const InputAttrPath & path); + std::shared_ptr findInput(const InputAttrPath & path) const; std::map getAllInputs() const; @@ -96,6 +96,34 @@ struct LockFileV7 void check(); }; -std::ostream & operator<<(std::ostream & stream, const LockFileV7 & lockFile); +struct LockedFlakeV7 : LockedFlake +{ + LockFileV7 lockFile; + + /** + * Source tree accessors for nodes that have been fetched in + * lockFlake(); in particular, the root node and the overridden + * inputs. + * FIXME: move into lockFile? + */ + std::map, SourcePath> nodePaths; + + LockedFlakeV7(Flake && flake, LockFileV7 && lockFile, std::map, SourcePath> && nodePaths) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + , nodePaths(std::move(nodePaths)) + { + } + + std::vector getInputNames(const InputAttrPath & prefix) const override; + + std::optional findInput(const InputAttrPath & path) const override; + + void visit(VisitCallback callback) const override; + + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; + + nlohmann::json toJSON() const override; +}; } // namespace nix::flake diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index d9230966dccf..50dd62c255b7 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -124,7 +124,7 @@ doFind(const ref & root, const InputAttrPath & path, std::vector LockFileV7::findInput(const InputAttrPath & path) +std::shared_ptr LockFileV7::findInput(const InputAttrPath & path) const { std::vector visited; return doFind(root, path, visited); @@ -253,12 +253,6 @@ std::pair LockFileV7::to_string() const return {json.dump(2), std::move(nodeKeys)}; } -std::ostream & operator<<(std::ostream & stream, const LockFileV7 & lockFile) -{ - stream << lockFile.toJSON().first.dump(2); - return stream; -} - std::optional LockFileV7::isUnlocked(const fetchers::Settings & fetchSettings) const { std::set> nodes; @@ -396,4 +390,58 @@ void LockFileV7::check() } } +std::vector LockedFlakeV7::getInputNames(const InputAttrPath & prefix) const +{ + std::vector res; + if (auto node = lockFile.findInput(prefix)) + for (auto & [id, input] : node->inputs) + res.push_back(id); + return res; +} + +std::optional LockedFlakeV7::findInput(const InputAttrPath & path) const +{ + if (auto node = std::dynamic_pointer_cast(lockFile.findInput(path))) + return InputInfo{ + .lockedRef = node->lockedRef, + .isFlake = node->isFlake, + }; + return std::nullopt; +} + +void LockedFlakeV7::visit(VisitCallback callback) const +{ + if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) + return; + + [&](this const auto & recurse, const InputAttrPath & prefix, ref node) -> void { + for (auto & [id, input] : node->inputs) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + if (auto child = std::get_if<0>(&input)) { + if (callback( + inputAttrPath, + InputInfo{ + .lockedRef = (*child)->lockedRef, + .isFlake = (*child)->isFlake, + .buildTime = (*child)->buildTime, + })) + recurse(inputAttrPath, *child); + } else if (auto follows = std::get_if<1>(&input)) { + callback(inputAttrPath, *follows); + } + } + }({}, lockFile.root); +} + +std::optional LockedFlakeV7::isUnlocked(const fetchers::Settings & fetchSettings) const +{ + return lockFile.isUnlocked(fetchSettings); +} + +nlohmann::json LockedFlakeV7::toJSON() const +{ + return lockFile.toJSON().first; +} + } // namespace nix::flake diff --git a/src/nix/flake-command.hh b/src/nix/flake-command.hh index ae52bcabb912..04e0801cfb30 100644 --- a/src/nix/flake-command.hh +++ b/src/nix/flake-command.hh @@ -17,7 +17,7 @@ public: FlakeRef getFlakeRef(); - flake::LockedFlake lockFlake(); + std::unique_ptr lockFlake(); std::vector getFlakeRefsForCompletion() override; }; diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index f4b944fcea53..9e9d684c2e07 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -24,46 +24,45 @@ struct CmdFlakePrefetchInputs : FlakeCommand void run(nix::ref store) override { - using namespace nix::flake; auto flake = lockFlake(); - ThreadPool pool{fileTransferSettings.httpConnections}; + /* Gather the locked references of all transitive inputs, + skipping build-time inputs and their dependencies. */ + std::vector lockedRefs; - struct State - { - std::set done; - }; + flake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + auto inputInfo = std::get_if(&input); - Sync state_; + /* Skip "follows" inputs and build-time inputs (and their + dependencies). */ + if (!inputInfo || inputInfo->buildTime) + return false; - std::atomic nrFailed{0}; + /* Skip the root flake, which we've fetched already. */ + if (!inputAttrPath.empty()) + lockedRefs.push_back(inputInfo->lockedRef); + + return true; + }); + + /* Fetch the inputs in parallel. */ + ThreadPool pool{fileTransferSettings.httpConnections}; - auto visit = [&](this const auto & visit, const Node & node) { - if (!state_.lock()->done.insert(&node).second) - return; + std::atomic nrFailed{0}; - if (auto lockedNode = dynamic_cast(&node)) { - if (lockedNode->buildTime) - return; + for (auto & lockedRef : lockedRefs) { + pool.enqueue([&, lockedRef]() { try { - Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); - auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first; + Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedRef)); + auto accessor = lockedRef.input.getAccessor(fetchSettings, *store).first; if (!evalSettings.lazyTrees) - fetchToStore( - fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); + fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); } catch (Error & e) { printError("%s", e.what()); nrFailed++; } - } - - for (auto & [inputName, input] : node.inputs) { - if (auto inputNode = std::get_if<0>(&input)) - pool.enqueue(std::bind(visit, **inputNode)); - } - }; - - pool.enqueue(std::bind(visit, *flake.lockFile.root)); + }); + } pool.process(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3deddb7dc6e6..6da6c650449e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -56,7 +56,7 @@ FlakeRef FlakeCommand::getFlakeRef() return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); // FIXME } -flake::LockedFlake FlakeCommand::lockFlake() +std::unique_ptr FlakeCommand::lockFlake() { return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } @@ -195,7 +195,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON { lockFlags.requireLockable = false; auto lockedFlake = lockFlake(); - auto & flake = lockedFlake.flake; + auto & flake = lockedFlake->flake; /* Hack to show the store path if available. */ std::optional storePath; @@ -227,8 +227,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; if (storePath) j["path"] = store->printStorePath(*storePath); - j["locks"] = lockedFlake.lockFile.toJSON().first; - if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) + j["locks"] = lockedFlake->toJSON(); + if (auto fingerprint = lockedFlake->getFingerprint(*store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); printJSON(j); } else { @@ -249,41 +249,56 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) + if (auto fingerprint = lockedFlake->getFingerprint(*store, fetchSettings)) logger->cout( ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false)); - if (!lockedFlake.lockFile.root->inputs.empty()) - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + /* Gather the inputs into a tree, since we need to know + the children of a node before we can print it. */ + struct TreeNode + { + std::optional> input; + std::map children; + }; + + TreeNode root; - std::set> visited{lockedFlake.lockFile.root}; + lockedFlake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + if (!inputAttrPath.empty()) { + auto * node = &root; + for (auto & elem : inputAttrPath) + node = &node->children[elem]; + node->input = input; + } + return true; + }); - [&](this const auto & recurse, const flake::Node & node, const std::string & prefix) -> void { - for (const auto & [last, input] : markLast(node.inputs)) { - if (auto lockedNode = std::get_if<0>(&input.second)) { + if (!root.children.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + + [&](this const auto & recurse, const TreeNode & node, const std::string & prefix) -> void { + for (const auto & [last, child] : markLast(node.children)) { + if (auto inputInfo = std::get_if(&*child.second.input)) { std::string lastModifiedStr = ""; - if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + if (auto lastModified = inputInfo->lockedRef.input.getLastModified()) lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); logger->cout( "%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), - input.first, - (*lockedNode)->lockedRef.to_string(true), + child.first, + inputInfo->lockedRef.to_string(true), lastModifiedStr); - bool firstVisit = visited.insert(*lockedNode).second; - - if (firstVisit) - recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); - } else if (auto follows = std::get_if<1>(&input.second)) { + recurse(child.second, prefix + (last ? treeNull : treeLine)); + } else if (auto follows = std::get_if(&*child.second.input)) { logger->cout( "%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), - input.first, + child.first, flake::printInputAttrPath(*follows)); } } - }(*lockedFlake.lockFile.root, ""); + }(root, ""); } } }; @@ -370,7 +385,7 @@ struct CmdFlakeCheck : FlakeCommand, MixPrintOutPaths, MixOutLinkBase, MixFlakeS auto state = getEvalState(); lockFlags.applyNixConfig = true; - auto flake = std::make_shared(lockFlake()); + std::shared_ptr flake(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); @@ -812,45 +827,42 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs StorePathSet sources; - auto storePath = dryRun ? flake.flake.lockedRef.input.computeStorePath(*store) - : std::get(flake.flake.lockedRef.input.fetchToStore(fetchSettings, *store)); - - sources.insert(storePath); - - // FIXME: use graph output, handle cycles. - auto traverse = - [&, json = json, dryRun = dryRun](this const auto & self, const flake::Node & node) -> nlohmann::json { - nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); - for (auto & [inputName, input] : node.inputs) { - if (auto inputNode = std::get_if<0>(&input)) { - std::optional storePath; - if (!(*inputNode)->lockedRef.input.isRelative()) { - storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : std::get( - (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store)); - sources.insert(*storePath); - } - if (json) { - auto & jsonObj3 = jsonObj2[inputName]; - if (storePath) - jsonObj3["path"] = store->printStorePath(*storePath); - jsonObj3["inputs"] = self(**inputNode); - } else - self(**inputNode); - } - } - return jsonObj2; + nlohmann::json jsonRoot; + + /* Return the JSON object for the input denoted by + `inputAttrPath`, creating it if necessary. */ + auto getJsonObj = [&](const flake::InputAttrPath & inputAttrPath) -> nlohmann::json & { + auto * jsonObj = &jsonRoot; + for (auto & elem : inputAttrPath) + jsonObj = &(*jsonObj)["inputs"][elem]; + return *jsonObj; }; - if (json) { - nlohmann::json jsonRoot = { - {"path", store->printStorePath(storePath)}, - {"inputs", traverse(*flake.lockFile.root)}, - }; + flake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + /* Skip "follows" inputs; their targets are visited under + their own paths. */ + auto inputInfo = std::get_if(&input); + if (!inputInfo) + return false; + + std::optional storePath; + if (!inputInfo->lockedRef.input.isRelative()) { + storePath = dryRun + ? inputInfo->lockedRef.input.computeStorePath(*store) + : std::get(inputInfo->lockedRef.input.fetchToStore(fetchSettings, *store)); + sources.insert(*storePath); + } + if (json) { + auto & jsonObj = getJsonObj(inputAttrPath); + if (storePath) + jsonObj["path"] = store->printStorePath(*storePath); + jsonObj["inputs"] = nlohmann::json::object(); + } + return true; + }); + + if (json) printJSON(jsonRoot); - } else { - traverse(*flake.lockFile.root); - } if (!dryRun && dstUri) { ref dstStore = openStore(StoreReference{*dstUri}); @@ -918,7 +930,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON, MixFlakeSchemas throw UsageError("The '--drv-paths' flag requires '--json'."); auto state = getEvalState(); - auto flake = make_ref(lockFlake()); + std::shared_ptr flake(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); From 05cf5550fbf60fb5e20799b2138067286507d1ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2026 16:23:20 +0200 Subject: [PATCH 05/26] Move the version 7 lock algorithm out of the free lockFlake() function The `computeLocks` machinery is moved from the free function `lockFlake()` in flake.cc into a new static member function `LockedFlakeV7::lockFlake()` in lockfile-v7.cc. The free function now only contains version-independent work, expressed in terms of the abstract `LockedFlake` interface: * Reading the old lock file into a `nlohmann::json`, and constructing a `LockedFlake` from it (via a new `LockedFlakeV7` constructor that takes the JSON). This is where we will dispatch on the lock file version in the future. * Change detection, by comparing the virtual `toJSON()` serializations of the old and new locked flakes. This also does the right thing across future version migrations (a version change is always a change). * Printing the diff, via a new abstract `LockedFlake::diff()` method that shows the differences relative to an older `LockedFlake`. The version 7 implementation diffs against an empty lock file if the old lock file is not version 7, so all inputs show up as added. * Writing/committing the new lock file, using `to_string()` and the virtual `isUnlocked()`. `LockedFlakeV7::lockFlake()` takes the old lock file as a `const LockedFlake &` and downcasts it internally. The `LockFileV7` parsing constructor now takes a `nlohmann::json` instead of the file contents, and `getFlake()` with a lock root attribute path is exported from flake.cc since the lock algorithm needs it from its new home. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 443 ++--------------- src/libflake/include/nix/flake/flake.hh | 13 + src/libflake/include/nix/flake/lockfile-v7.hh | 25 +- src/libflake/lockfile-v7.cc | 453 +++++++++++++++++- 4 files changed, 506 insertions(+), 428 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index dce43df6a922..f24914577598 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -373,7 +373,7 @@ static FlakeRef applySelfAttrs(const FlakeRef & ref, const Flake & flake) return newRef; } -static Flake getFlake( +Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, @@ -420,441 +420,56 @@ Flake getFlake( return getFlake(state, originalRef, useRegistries, {}, requireLockable); } -static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) -{ - return lockFilePath.pathExists() ? LockFileV7(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) - : LockFileV7(); -} - std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; - auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; if (lockFlags.applyNixConfig) { flake.config.apply(settings); state.store->setOptions(); } + auto flakeRefForTrace = flake.lockedRef.to_string(); + try { if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } - auto oldLockFile = - readLockFile(state.fetchSettings, lockFlags.referenceLockFilePath.value_or(flake.lockFilePath())); - - debug("old lock file: %s", oldLockFile.to_string().first); - - struct OverrideTarget - { - FlakeInput input; - SourcePath sourcePath; - std::optional parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix? - }; - - std::map overrides; - std::set explicitCliOverrides; - std::set overridesUsed; - std::set updatesUsed; - std::map, SourcePath> nodePaths; - - for (auto & i : lockFlags.inputOverrides) { - overrides.emplace( - i.first, - OverrideTarget{ - .input = FlakeInput{.ref = i.second}, - /* Note: any relative overrides - (e.g. `--override-input B/C "path:./foo/bar"`) - are interpreted relative to the top-level - flake. */ - .sourcePath = flake.path, - }); - explicitCliOverrides.insert(i.first); - } + auto lockFilePath = lockFlags.referenceLockFilePath.value_or(flake.lockFilePath()); - LockFileV7 newLockFile; - - std::vector parents; - - std::function node, - const InputAttrPath & inputAttrPathPrefix, - std::shared_ptr oldNode, - const InputAttrPath & followsPrefix, - const SourcePath & sourcePath, - bool trustLock)> - computeLocks; - - computeLocks = [&]( - /* The inputs of this node, either from flake.nix or - flake.lock. */ - const FlakeInputs & flakeInputs, - /* The node whose locks are to be updated.*/ - ref node, - /* The path to this node in the lock file graph. */ - const InputAttrPath & inputAttrPathPrefix, - /* The old node, if any, from which locks can be - copied. */ - std::shared_ptr oldNode, - /* The prefix relative to which 'follows' should be - interpreted. When a node is initially locked, it's - relative to the node's flake; when it's already locked, - it's relative to the root of the lock file. */ - const InputAttrPath & followsPrefix, - /* The source path of this node's flake. */ - const SourcePath & sourcePath, - bool trustLock) { - debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix)); - - /* Get the overrides (i.e. attributes of the form - 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - auto addOverrides = - [&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void { - for (auto & [idOverride, inputOverride] : input.overrides) { - auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride); - if (inputOverride.ref || inputOverride.follows) - overrides.emplace( - inputAttrPath, - OverrideTarget{ - .input = inputOverride, - .sourcePath = sourcePath, - .parentInputAttrPath = inputAttrPathPrefix}); - addOverrides(inputOverride, inputAttrPath); - } - }; + nlohmann::json oldLockFileJson; - for (auto & [id, input] : flakeInputs) { - auto inputAttrPath(inputAttrPathPrefix); - inputAttrPath.push_back(id); - addOverrides(input, inputAttrPath); + if (lockFilePath.pathExists()) { + try { + oldLockFileJson = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); } + } - /* Check whether this input has overrides for a - non-existent input. */ - for (auto [inputAttrPath, inputOverride] : overrides) { - auto follow = inputAttrPath.inputName(); - auto inputAttrPath2 = inputAttrPath.parent(); - if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow)) - warn( - "input '%s' has an override for a non-existent input '%s'", - printInputAttrPath(inputAttrPathPrefix), - follow); - } - - /* Go over the flake inputs, resolve/fetch them if - necessary (i.e. if they're new or the flakeref changed - from what's in the lock file). */ - for (auto & [id, input2] : flakeInputs) { - auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id); - auto inputAttrPath = nonEmptyInputAttrPath.get(); - auto inputAttrPathS = printInputAttrPath(inputAttrPath); - debug("computing input '%s'", inputAttrPathS); - - try { - - /* Do we have an override for this input from one of the - ancestors? */ - auto i = overrides.find(nonEmptyInputAttrPath); - bool hasOverride = i != overrides.end(); - bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); - if (hasOverride) - overridesUsed.insert(nonEmptyInputAttrPath); - auto input = hasOverride ? i->second.input : input2; - - /* Resolve relative 'path:' inputs relative to - the source path of the overrider. */ - auto overriddenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; - - /* Respect the "flakeness" of the input even if we - override it. */ - if (hasOverride) - input.isFlake = input2.isFlake; - - /* Resolve 'follows' later (since it may refer to an input - path we haven't processed yet. */ - if (input.follows) { - InputAttrPath target; - - target.insert(target.end(), input.follows->begin(), input.follows->end()); - - debug("input '%s' follows '%s'", inputAttrPathS, printInputAttrPath(target)); - node->inputs.insert_or_assign(id, target); - continue; - } - - if (!input.ref) - input.ref = - FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}}); - - auto overriddenParentPath = - input.ref->input.isRelative() - ? std::optional( - hasOverride ? i->second.parentInputAttrPath : inputAttrPathPrefix) - : std::nullopt; - - auto resolveRelativePath = [&]() -> std::optional { - if (auto relativePath = input.ref->input.isRelative()) { - return SourcePath{ - overriddenSourcePath.accessor, - CanonPath(relativePath->string(), overriddenSourcePath.path.parent().value())}; - } else - return std::nullopt; - }; - - /* Get the input flake, resolve 'path:./...' - flakerefs relative to the parent flake. */ - auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { - if (auto resolvedPath = resolveRelativePath()) { - return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); - } else { - return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); - } - }; - - /* Do we have an entry in the existing lock file? - And the input is not in updateInputs? */ - std::shared_ptr oldLock; - - updatesUsed.insert(inputAttrPath); - - if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath)) - if (auto oldLock2 = get(oldNode->inputs, id)) - if (auto oldLock3 = std::get_if<0>(&*oldLock2)) - oldLock = *oldLock3; - - if (oldLock && oldLock->originalRef.canonicalize() == input.ref->canonicalize() - && oldLock->parentInputAttrPath == overriddenParentPath && !hasCliOverride) { - debug("keeping existing input '%s'", inputAttrPathS); - - /* Copy the input from the old lock since its flakeref - didn't change and there is no override from a - higher level flake. */ - auto childNode = make_ref( - oldLock->lockedRef, - oldLock->originalRef, - oldLock->isFlake, - oldLock->buildTime, - oldLock->parentInputAttrPath); - - node->inputs.insert_or_assign(id, childNode); - - /* If we have this input in updateInputs, then we - must fetch the flake to update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath); - - auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size() - && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); - - FlakeInputs fakeInputs; - - if (!mustRefetch) { - /* No need to fetch this flake, we can be - lazy. However there may be new overrides on the - inputs of this flake, so we need to check - those. */ - for (auto & i : oldLock->inputs) { - if (auto lockedNode = std::get_if<0>(&i.second)) { - fakeInputs.emplace( - i.first, - FlakeInput{ - .ref = (*lockedNode)->originalRef, - .isFlake = (*lockedNode)->isFlake, - }); - } else if (auto follows = std::get_if<1>(&i.second)) { - if (!trustLock) { - // It is possible that the flake has changed, - // so we must confirm all the follows that are in the lock file are also in the - // flake. - auto overridePath = - NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first); - auto o = overrides.find(overridePath); - // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lock file. - if (o == overrides.end()) { - mustRefetch = true; - // There's no point populating the rest of the fake inputs, - // since we'll refetch the flake anyways. - break; - } - } - auto absoluteFollows(followsPrefix); - absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); - fakeInputs.emplace( - i.first, - FlakeInput{ - .follows = absoluteFollows, - }); - } - } - } - - if (mustRefetch) { - auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); - nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks( - inputFlake.inputs, - childNode, - inputAttrPath, - oldLock, - followsPrefix, - inputFlake.path, - false); - } else { - computeLocks( - fakeInputs, childNode, inputAttrPath, oldLock, followsPrefix, sourcePath, true); - } - - } else { - /* We need to create a new lock file entry. So fetch - this input. */ - debug("creating new input '%s'", inputAttrPathS); - - if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) - && !input.ref->input.isRelative()) - throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); - - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); - auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; - - /* Warn against the use of indirect flakerefs - (but only at top-level since we don't want - to annoy users about flakes that are not - under their control). */ - auto warnRegistry = [&](const FlakeRef & resolvedRef) { - if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { - std::ostringstream s; - printLiteralString(s, resolvedRef.to_string()); - warn( - "Flake input '%1%' uses the flake registry. " - "Using the registry in flake inputs is deprecated in Determinate Nix. " - "To make your flake future-proof, add the following to '%2%':\n" - "\n" - " inputs.%1%.url = %3%;\n" - "\n" - "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", - inputAttrPathS, - flake.path, - s.str()); - } - }; - - if (input.isFlake) { - auto inputFlake = getInputFlake( - *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); - - auto childNode = make_ref( - inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); - - node->inputs.insert_or_assign(id, childNode); - - /* Guard against circular flake imports. */ - for (auto & parent : parents) - if (parent == *input.ref) - throw Error("found circular import of flake '%s'", parent); - parents.push_back(*input.ref); - Finally cleanup([&]() { parents.pop_back(); }); - - /* Recursively process the inputs of this - flake, using its own lock file. */ - nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks( - inputFlake.inputs, - childNode, - inputAttrPath, - readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), - inputAttrPath, - inputFlake.path, - false); - - warnRegistry(inputFlake.resolvedRef); - } - - else { - auto [path, lockedRef] = [&]() -> std::tuple { - // Handle non-flake 'path:./...' inputs. - if (auto resolvedPath = resolveRelativePath()) { - return {*resolvedPath, *input.ref}; - } else { - auto cachedInput = state.inputCache->getAccessor( - state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); - - auto resolvedRef = - FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); - auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - - warnRegistry(resolvedRef); - - return { - state.storePath(state.mountInput( - lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), - lockedRef}; - } - }(); - - auto childNode = - make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); - - nodePaths.emplace(childNode, path); - - node->inputs.insert_or_assign(id, childNode); - } - } - - } catch (Error & e) { - e.addTrace({}, "while updating the flake input '%s'", inputAttrPathS); - throw; - } - } - }; - - nodePaths.emplace(newLockFile.root, flake.path.parent()); - - computeLocks( - flake.inputs, - newLockFile.root, - {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), - {}, - flake.path, - false); - - for (auto & i : lockFlags.inputOverrides) - if (!overridesUsed.count(i.first)) - warn( - "the flag '--override-input %s %s' does not match any input", - printInputAttrPath(i.first), - i.second); + // FIXME: dispatch on the lock file version here. + LockedFlakeV7 oldLockedFlake(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); - for (auto & i : lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("'%s' does not match any input of this flake", printInputAttrPath(i)); + debug("old lock file: %s", oldLockedFlake.to_string()); - /* Check 'follows' inputs. */ - newLockFile.check(); + auto lockedFlake = LockedFlakeV7::lockFlake(settings, state, lockFlags, std::move(flake), oldLockedFlake); - debug("new lock file: %s", newLockFile.to_string().first); + debug("new lock file: %s", lockedFlake->to_string()); auto sourcePath = topRef.input.getSourcePath(); /* Check whether we need to / can write the new lock file. */ - if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { + auto lockedFlakeJson = lockedFlake->toJSON(); + if (lockedFlakeJson != oldLockedFlake.toJSON() || lockFlags.outputLockFilePath) { - auto diff = LockFileV7::diff(oldLockFile, newLockFile); + auto diff = lockedFlake->diff(oldLockedFlake); if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { - if (auto unlockedInput = newLockFile.isUnlocked(state.fetchSettings)) { + if (auto unlockedInput = lockedFlake->isUnlocked(state.fetchSettings)) { if (lockFlags.failOnUnlocked) throw Error( "Not writing lock file of flake '%s' because it has an unlocked input ('%s'). " @@ -872,7 +487,7 @@ std::unique_ptr lockFlake( "flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto newLockFileS = fmt("%s\n", newLockFile.to_string().first); + auto newLockFileS = fmt("%s\n", lockedFlakeJson.dump(2)); if (lockFlags.outputLockFilePath) { if (lockFlags.commitLockFile) @@ -918,26 +533,26 @@ std::unique_ptr lockFlake( /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ - auto prevLockedRef = flake.lockedRef; - flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); + auto prevLockedRef = lockedFlake->flake.lockedRef; + lockedFlake->flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); - if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() - && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) - warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + if (lockFlags.commitLockFile && lockedFlake->flake.lockedRef.input.getRev() + && prevLockedRef.input.getRev() != lockedFlake->flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", lockedFlake->flake.lockedRef.input.getRev()->gitRev()); } } else throw Error( "cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); } else { warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); - flake.forceDirty = true; + lockedFlake->flake.forceDirty = true; } } - return std::make_unique(std::move(flake), std::move(newLockFile), std::move(nodePaths)); + return lockedFlake; } catch (Error & e) { - e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); + e.addTrace({}, "while updating the lock file of flake '%s'", flakeRefForTrace); throw; } } diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 3b1b0006e8dd..ec2692f1e468 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -134,6 +134,13 @@ struct Flake Flake getFlake( EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries, bool requireLockable = true); +Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + fetchers::UseRegistries useRegistries, + const InputAttrPath & lockRootAttrPath, + bool requireLockable); + /** * Fingerprint of a locked flake; used as a cache key. */ @@ -201,6 +208,12 @@ struct LockedFlake */ virtual std::optional isUnlocked(const fetchers::Settings & fetchSettings) const = 0; + /** + * Return a human-readable description of the differences between + * the (older) `oldLockFile` and the lock file of this flake. + */ + virtual std::string diff(const LockedFlake & oldLockFile) const = 0; + virtual nlohmann::json toJSON() const = 0; std::string to_string() const; diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index ed67903a4175..9c9808337bc2 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -68,7 +68,7 @@ struct LockFileV7 ref root = make_ref(); LockFileV7() {}; - LockFileV7(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path); + LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path); typedef std::map, std::string> KeyMap; @@ -88,8 +88,6 @@ struct LockFileV7 std::map getAllInputs() const; - static std::string diff(const LockFileV7 & oldLocks, const LockFileV7 & newLocks); - /** * Check that every 'follows' input target exists. */ @@ -115,6 +113,13 @@ struct LockedFlakeV7 : LockedFlake { } + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); + std::vector getInputNames(const InputAttrPath & prefix) const override; std::optional findInput(const InputAttrPath & path) const override; @@ -123,7 +128,21 @@ struct LockedFlakeV7 : LockedFlake std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; + std::string diff(const LockedFlake & oldLockFile) const override; + nlohmann::json toJSON() const override; + + /** + * Compute a lock file for `flake`, reusing entries from + * `oldLockFile` (which must be a `LockedFlakeV7`) where + * possible. Note: this does not write the new lock file. + */ + static std::unique_ptr lockFlake( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); }; } // namespace nix::flake diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 50dd62c255b7..09fecc72b9c3 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -22,6 +22,13 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/lockfile-v7.hh" +#include "nix/flake/settings.hh" +#include "nix/expr/eval.hh" +#include "nix/expr/nixexpr.hh" +#include "nix/store/store-api.hh" +#include "nix/fetchers/input-cache.hh" +#include "nix/util/finally.hh" +#include "nix/util/canon-path.hh" #include "nix/util/strings.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" @@ -130,15 +137,8 @@ std::shared_ptr LockFileV7::findInput(const InputAttrPath & path) const return doFind(root, path, visited); } -LockFileV7::LockFileV7(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path) +LockFileV7::LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) { - auto json = [=] { - try { - return nlohmann::json::parse(contents); - } catch (const nlohmann::json::parse_error & e) { - throw Error("Could not parse '%s': %s", path, e.what()); - } - }(); auto version = json.value("version", 0); if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); @@ -342,10 +342,15 @@ static bool equals(const Node::Edge & e1, const Node::Edge & e2) return false; } -std::string LockFileV7::diff(const LockFileV7 & oldLocks, const LockFileV7 & newLocks) +std::string LockedFlakeV7::diff(const LockedFlake & _oldLockFile) const { - auto oldFlat = oldLocks.getAllInputs(); - auto newFlat = newLocks.getAllInputs(); + /* If `oldLockFile` is not a version 7 lock file, diff against an + empty lock file, i.e. all inputs of this lock file will show up + as added. */ + auto oldLockFile = dynamic_cast(&_oldLockFile); + + auto oldFlat = oldLockFile ? oldLockFile->lockFile.getAllInputs() : std::map(); + auto newFlat = lockFile.getAllInputs(); auto i = oldFlat.begin(); auto j = newFlat.begin(); @@ -444,4 +449,430 @@ nlohmann::json LockedFlakeV7::toJSON() const return lockFile.toJSON().first; } +LockedFlakeV7::LockedFlakeV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) +{ +} + +static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) +{ + if (!lockFilePath.pathExists()) + return LockFileV7(); + + auto json = [&] { + try { + return nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + }(); + + return LockFileV7(fetchSettings, json, fmt("%s", lockFilePath)); +} + +std::unique_ptr LockedFlakeV7::lockFlake( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & _oldLockFile) +{ + auto & oldLockFile = dynamic_cast(_oldLockFile).lockFile; + + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; + + struct OverrideTarget + { + FlakeInput input; + SourcePath sourcePath; + std::optional parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix? + }; + + std::map overrides; + std::set explicitCliOverrides; + std::set overridesUsed; + std::set updatesUsed; + std::map, SourcePath> nodePaths; + + for (auto & i : lockFlags.inputOverrides) { + overrides.emplace( + i.first, + OverrideTarget{ + .input = FlakeInput{.ref = i.second}, + /* Note: any relative overrides + (e.g. `--override-input B/C "path:./foo/bar"`) + are interpreted relative to the top-level + flake. */ + .sourcePath = flake.path, + }); + explicitCliOverrides.insert(i.first); + } + + LockFileV7 newLockFile; + + std::vector parents; + + std::function node, + const InputAttrPath & inputAttrPathPrefix, + std::shared_ptr oldNode, + const InputAttrPath & followsPrefix, + const SourcePath & sourcePath, + bool trustLock)> + computeLocks; + + computeLocks = [&]( + /* The inputs of this node, either from flake.nix or + flake.lock. */ + const FlakeInputs & flakeInputs, + /* The node whose locks are to be updated.*/ + ref node, + /* The path to this node in the lock file graph. */ + const InputAttrPath & inputAttrPathPrefix, + /* The old node, if any, from which locks can be + copied. */ + std::shared_ptr oldNode, + /* The prefix relative to which 'follows' should be + interpreted. When a node is initially locked, it's + relative to the node's flake; when it's already locked, + it's relative to the root of the lock file. */ + const InputAttrPath & followsPrefix, + /* The source path of this node's flake. */ + const SourcePath & sourcePath, + bool trustLock) { + debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + auto addOverrides = + [&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride); + if (inputOverride.ref || inputOverride.follows) + overrides.emplace( + inputAttrPath, + OverrideTarget{ + .input = inputOverride, + .sourcePath = sourcePath, + .parentInputAttrPath = inputAttrPathPrefix}); + addOverrides(inputOverride, inputAttrPath); + } + }; + + for (auto & [id, input] : flakeInputs) { + auto inputAttrPath(inputAttrPathPrefix); + inputAttrPath.push_back(id); + addOverrides(input, inputAttrPath); + } + + /* Check whether this input has overrides for a + non-existent input. */ + for (auto [inputAttrPath, inputOverride] : overrides) { + auto follow = inputAttrPath.inputName(); + auto inputAttrPath2 = inputAttrPath.parent(); + if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow)) + warn( + "input '%s' has an override for a non-existent input '%s'", + printInputAttrPath(inputAttrPathPrefix), + follow); + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id); + auto inputAttrPath = nonEmptyInputAttrPath.get(); + auto inputAttrPathS = printInputAttrPath(inputAttrPath); + debug("computing input '%s'", inputAttrPathS); + + try { + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(nonEmptyInputAttrPath); + bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); + if (hasOverride) + overridesUsed.insert(nonEmptyInputAttrPath); + auto input = hasOverride ? i->second.input : input2; + + /* Resolve relative 'path:' inputs relative to + the source path of the overrider. */ + auto overriddenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; + + /* Respect the "flakeness" of the input even if we + override it. */ + if (hasOverride) + input.isFlake = input2.isFlake; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputAttrPath target; + + target.insert(target.end(), input.follows->begin(), input.follows->end()); + + debug("input '%s' follows '%s'", inputAttrPathS, printInputAttrPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } + + if (!input.ref) + input.ref = + FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}}); + + auto overriddenParentPath = input.ref->input.isRelative() + ? std::optional( + hasOverride ? i->second.parentInputAttrPath : inputAttrPathPrefix) + : std::nullopt; + + auto resolveRelativePath = [&]() -> std::optional { + if (auto relativePath = input.ref->input.isRelative()) { + return SourcePath{ + overriddenSourcePath.accessor, + CanonPath(relativePath->string(), overriddenSourcePath.path.parent().value())}; + } else + return std::nullopt; + }; + + /* Get the input flake, resolve 'path:./...' + flakerefs relative to the parent flake. */ + auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { + if (auto resolvedPath = resolveRelativePath()) { + return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); + } else { + return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); + } + }; + + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ + std::shared_ptr oldLock; + + updatesUsed.insert(inputAttrPath); + + if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; + + if (oldLock && oldLock->originalRef.canonicalize() == input.ref->canonicalize() + && oldLock->parentInputAttrPath == overriddenParentPath && !hasCliOverride) { + debug("keeping existing input '%s'", inputAttrPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = make_ref( + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->buildTime, + oldLock->parentInputAttrPath); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath); + + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size() + && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); + + FlakeInputs fakeInputs; + + if (!mustRefetch) { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace( + i.first, + FlakeInput{ + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + if (!trustLock) { + // It is possible that the flake has changed, + // so we must confirm all the follows that are in the lock file are also in the + // flake. + auto overridePath = NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first); + auto o = overrides.find(overridePath); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lock file. + if (o == overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } + } + auto absoluteFollows(followsPrefix); + absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); + fakeInputs.emplace( + i.first, + FlakeInput{ + .follows = absoluteFollows, + }); + } + } + } + + if (mustRefetch) { + auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); + nodePaths.emplace(childNode, inputFlake.path.parent()); + computeLocks( + inputFlake.inputs, + childNode, + inputAttrPath, + oldLock, + followsPrefix, + inputFlake.path, + false); + } else { + computeLocks(fakeInputs, childNode, inputAttrPath, oldLock, followsPrefix, sourcePath, true); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputAttrPathS); + + if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) + && !input.ref->input.isRelative()) + throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); + auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; + + /* Warn against the use of indirect flakerefs + (but only at top-level since we don't want + to annoy users about flakes that are not + under their control). */ + auto warnRegistry = [&](const FlakeRef & resolvedRef) { + if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + inputAttrPathS, + flake.path, + s.str()); + } + }; + + if (input.isFlake) { + auto inputFlake = getInputFlake( + *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); + + auto childNode = make_ref( + inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake, using its own lock file. */ + nodePaths.emplace(childNode, inputFlake.path.parent()); + computeLocks( + inputFlake.inputs, + childNode, + inputAttrPath, + readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + inputAttrPath, + inputFlake.path, + false); + + warnRegistry(inputFlake.resolvedRef); + } + + else { + auto [path, lockedRef] = [&]() -> std::tuple { + // Handle non-flake 'path:./...' inputs. + if (auto resolvedPath = resolveRelativePath()) { + return {*resolvedPath, *input.ref}; + } else { + auto cachedInput = state.inputCache->getAccessor( + state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); + + warnRegistry(resolvedRef); + + return { + state.storePath(state.mountInput( + lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), + lockedRef}; + } + }(); + + auto childNode = + make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); + + nodePaths.emplace(childNode, path); + + node->inputs.insert_or_assign(id, childNode); + } + } + + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", inputAttrPathS); + throw; + } + } + }; + + nodePaths.emplace(newLockFile.root, flake.path.parent()); + + computeLocks( + flake.inputs, + newLockFile.root, + {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), + {}, + flake.path, + false); + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", printInputAttrPath(i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("'%s' does not match any input of this flake", printInputAttrPath(i)); + + /* Check 'follows' inputs. */ + newLockFile.check(); + + return std::make_unique(std::move(flake), std::move(newLockFile), std::move(nodePaths)); +} + } // namespace nix::flake From 737742277891bc23daeb89fe2047d20bab46f018 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jul 2026 19:15:47 +0200 Subject: [PATCH 06/26] Add LockedFlake::getSourcePath() This abstract method returns the source path of the input denoted by an input attribute path (or of the top-level flake if the path is empty), fetching the input if necessary. The returned path is backed by `EvalState::rootFS`, i.e. it's a store path - possibly a virtual one with the input's accessor mounted on it if lazy trees are enabled. `nix flake prefetch-inputs` now uses this method instead of accessing the version 7 node graph directly, so when lazy trees are disabled, inputs are copied to the store via `EvalState::mountInput()` (which also verifies their NAR hashes). In `LockedFlakeV7`, the `nodePaths` map is replaced by a `mutable Sync>` field in `LockedNode`, i.e. the source path of a fetched node is now stored in the node itself. It is set by `LockedFlakeV7::lockFlake()` for nodes fetched during locking, and cached by `getSourcePath()` for nodes fetched on demand (where relative path inputs are resolved against the source path of their parent flake, and the `subdir` of the locked flakeref is taken into account). `callFlake()` reconstructs the node -> source path map by walking the node graph. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 20 ++++++- src/libflake/include/nix/flake/flake.hh | 10 ++++ src/libflake/include/nix/flake/lockfile-v7.hh | 22 +++---- src/libflake/lockfile-v7.cc | 57 ++++++++++++++++--- src/nix/flake-prefetch-inputs.cc | 20 ++++--- 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index f24914577598..b6fa5bd3ab59 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -601,9 +601,25 @@ void callFlake(EvalState & state, const LockedFlake & _lockedFlake, Value & vRes auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); - auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); + /* Gather the source paths of the nodes that have been fetched + (i.e. the root and the nodes fetched during locking). */ + std::map, SourcePath> nodePaths; + + nodePaths.emplace(lockedFlake.lockFile.root, lockedFlake.flake.path.parent()); + + [&](this const auto & recurse, ref node) -> void { + for (auto & [id, input] : node->inputs) { + if (auto child = std::get_if<0>(&input)) { + if (auto sourcePath = *(*child)->sourcePath.lock()) + nodePaths.emplace(*child, *sourcePath); + recurse(*child); + } + } + }(lockedFlake.lockFile.root); + + auto overrides = state.buildBindings(nodePaths.size()); - for (auto & [node, sourcePath] : lockedFlake.nodePaths) { + for (auto & [node, sourcePath] : nodePaths) { auto override = state.buildBindings(2); auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index ec2692f1e468..ac21388d652e 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -180,6 +180,16 @@ struct LockedFlake */ virtual std::optional findInput(const InputAttrPath & path) const = 0; + /** + * Return the source path of the input denoted by `inputAttrPath` + * (or of the top-level flake if `inputAttrPath` is empty), + * fetching it if necessary. Note: the returned path is backed by + * `EvalState::rootFS` (i.e. it's a store path, possibly a virtual + * one that has the input's accessor mounted on it if lazy trees + * are enabled), not by the input's original accessor. + */ + virtual SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const = 0; + /** * Callback for `visit()`. The second argument is either an * `InputInfo` for locked inputs, or, for "follows" inputs, the diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index 9c9808337bc2..13ad062c5550 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -2,6 +2,7 @@ ///@file #include "nix/flake/flake.hh" +#include "nix/util/sync.hh" #include @@ -41,6 +42,14 @@ struct LockedNode : Node (e.g. 'path:../foo') are interpreted. */ std::optional parentInputAttrPath; + /** + * The source path of this node, if it has been fetched. Set by + * `LockedFlakeV7::lockFlake()` for nodes fetched during locking, + * and by `LockedFlakeV7::getSourcePath()` for nodes fetched on + * demand. + */ + mutable Sync> sourcePath; + LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, @@ -98,18 +107,9 @@ struct LockedFlakeV7 : LockedFlake { LockFileV7 lockFile; - /** - * Source tree accessors for nodes that have been fetched in - * lockFlake(); in particular, the root node and the overridden - * inputs. - * FIXME: move into lockFile? - */ - std::map, SourcePath> nodePaths; - - LockedFlakeV7(Flake && flake, LockFileV7 && lockFile, std::map, SourcePath> && nodePaths) + LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) : LockedFlake(std::move(flake)) , lockFile(std::move(lockFile)) - , nodePaths(std::move(nodePaths)) { } @@ -124,6 +124,8 @@ struct LockedFlakeV7 : LockedFlake std::optional findInput(const InputAttrPath & path) const override; + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override; + void visit(VisitCallback callback) const override; std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 09fecc72b9c3..99ad7c4a5168 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -414,6 +414,52 @@ std::optional LockedFlakeV7::findInput(const InputAttrPa return std::nullopt; } +SourcePath LockedFlakeV7::getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const +{ + /* The root node. */ + if (inputAttrPath.empty()) + return flake.path.parent(); + + auto node = lockFile.findInput(inputAttrPath); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + + auto lockedNode = std::dynamic_pointer_cast(node); + assert(lockedNode); + + { + auto sourcePath(lockedNode->sourcePath.lock()); + if (*sourcePath) + return **sourcePath; + } + + /* Note: we fetch without holding the `sourcePath` lock, so + concurrent calls don't get serialized. Racing fetches of the + same node are harmless since they produce the same path. */ + auto path = [&]() -> SourcePath { + if (auto relativePath = lockedNode->lockedRef.input.isRelative()) { + /* Resolve relative path inputs against the source path of + their parent flake. */ + auto parentPath = getSourcePath(state, lockedNode->parentInputAttrPath.value()); + return {parentPath.accessor, CanonPath(relativePath->string(), parentPath.path)}; + } else { + /* Note: `lockedRef` is a copy since `mountInput()` may + modify the input (e.g. adding a `narHash` attribute). */ + auto lockedRef = lockedNode->lockedRef; + auto accessor = + state.inputCache + ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) + .accessor; + return state.storePath(state.mountInput(lockedRef.input, lockedNode->lockedRef.input, accessor, true, true)) + / CanonPath(lockedRef.subdir); + } + }(); + + *lockedNode->sourcePath.lock() = path; + + return path; +} + void LockedFlakeV7::visit(VisitCallback callback) const { if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) @@ -495,7 +541,6 @@ std::unique_ptr LockedFlakeV7::lockFlake( std::set explicitCliOverrides; std::set overridesUsed; std::set updatesUsed; - std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { overrides.emplace( @@ -728,7 +773,7 @@ std::unique_ptr LockedFlakeV7::lockFlake( if (mustRefetch) { auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); - nodePaths.emplace(childNode, inputFlake.path.parent()); + *childNode->sourcePath.lock() = inputFlake.path.parent(); computeLocks( inputFlake.inputs, childNode, @@ -800,7 +845,7 @@ std::unique_ptr LockedFlakeV7::lockFlake( /* Recursively process the inputs of this flake, using its own lock file. */ - nodePaths.emplace(childNode, inputFlake.path.parent()); + *childNode->sourcePath.lock() = inputFlake.path.parent(); computeLocks( inputFlake.inputs, childNode, @@ -837,7 +882,7 @@ std::unique_ptr LockedFlakeV7::lockFlake( auto childNode = make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); - nodePaths.emplace(childNode, path); + *childNode->sourcePath.lock() = path; node->inputs.insert_or_assign(id, childNode); } @@ -850,8 +895,6 @@ std::unique_ptr LockedFlakeV7::lockFlake( } }; - nodePaths.emplace(newLockFile.root, flake.path.parent()); - computeLocks( flake.inputs, newLockFile.root, @@ -872,7 +915,7 @@ std::unique_ptr LockedFlakeV7::lockFlake( /* Check 'follows' inputs. */ newLockFile.check(); - return std::make_unique(std::move(flake), std::move(newLockFile), std::move(nodePaths)); + return std::make_unique(std::move(flake), std::move(newLockFile)); } } // namespace nix::flake diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index 9e9d684c2e07..e85518a4eb69 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -1,5 +1,4 @@ #include "flake-command.hh" -#include "nix/fetchers/fetch-to-store.hh" #include "nix/util/thread-pool.hh" #include "nix/store/filetransfer.hh" #include "nix/util/exit.hh" @@ -26,9 +25,9 @@ struct CmdFlakePrefetchInputs : FlakeCommand { auto flake = lockFlake(); - /* Gather the locked references of all transitive inputs, + /* Gather the attribute paths of all transitive inputs, skipping build-time inputs and their dependencies. */ - std::vector lockedRefs; + std::vector> inputs; flake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { auto inputInfo = std::get_if(&input); @@ -40,23 +39,26 @@ struct CmdFlakePrefetchInputs : FlakeCommand /* Skip the root flake, which we've fetched already. */ if (!inputAttrPath.empty()) - lockedRefs.push_back(inputInfo->lockedRef); + inputs.emplace_back(inputAttrPath, inputInfo->lockedRef); return true; }); + auto state = getEvalState(); + /* Fetch the inputs in parallel. */ ThreadPool pool{fileTransferSettings.httpConnections}; std::atomic nrFailed{0}; - for (auto & lockedRef : lockedRefs) { - pool.enqueue([&, lockedRef]() { + for (auto & [inputAttrPath, lockedRef] : inputs) { + pool.enqueue([&, inputAttrPath, lockedRef]() { try { Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedRef)); - auto accessor = lockedRef.input.getAccessor(fetchSettings, *store).first; - if (!evalSettings.lazyTrees) - fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); + /* Note: when lazy trees are disabled, this also + copies the input to the store (via + `EvalState::mountInput()`). */ + flake->getSourcePath(*state, inputAttrPath); } catch (Error & e) { printError("%s", e.what()); nrFailed++; From b06972aca800452221322916f71df3875a06628b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2026 16:21:46 +0200 Subject: [PATCH 07/26] Make call-flake.nix independent of the lock file format Instead of the version 7 lock file JSON and an attrset of pre-fetched source trees ("overrides"), call-flake.nix now receives an external value wrapping the C++ `LockedFlake` object, along with two internal primops (registered with `internal = true`, so they're not exposed to the user): * `listFlakeInputs lockedFlake inputAttrPath`: returns the inputs of the input denoted by `inputAttrPath` as an attrset mapping input names to either null (for a regular input) or the input attribute path of the target of a "follows" input. This only consults the lock data, so it doesn't fetch anything. * `fetchFlakeInput lockedFlake inputAttrPath`: fetches an input (via `LockedFlake::getSourcePath()`) and returns its `sourceInfo` attributes and flake subdirectory; or, for build-time inputs, the locked input attributes (without fetching). As a result, fetching, override handling, relative path handling and the lock file node graph all disappear from call-flake.nix - it just lazily constructs a tree of inputs keyed by input attribute path, resolving "follows" by walking the edges of that tree from the top-level flake. This preserves evaluation sharing (every distinct input is constructed only once) and prepares for the version 8 lock file format, whose `LockedFlake` implementation will only need to provide the same abstract interface. Also: * `callFlake()` now takes a `std::shared_ptr` since the thunks created by call-flake.nix reference the `LockedFlake` object, which therefore must be kept alive for the lifetime of the evaluator. * `LockedFlake::getInputTargets()` is a new abstract method backing `listFlakeInputs`; `getInputNames()` is now a non-virtual wrapper around it. * `LockedFlakeV7::findInput()` now returns the `buildTime` and `parentInputAttrPath` fields (the former was dropped before, but wasn't used). * The `KeyMap` returned by `LockFileV7::toJSON()` is no longer needed, since lock file node keys were only used by callFlake(). Assisted-by: Claude Fable 5 --- src/libcmd/flake-schemas.cc | 4 +- src/libcmd/repl.cc | 2 +- src/libflake-c/nix_api_flake.cc | 2 +- src/libflake/call-flake.nix | 112 +++++----- src/libflake/flake-primops.cc | 6 +- src/libflake/flake.cc | 195 +++++++++++++++--- src/libflake/include/nix/flake/flake.hh | 22 +- src/libflake/include/nix/flake/lockfile-v7.hh | 8 +- src/libflake/lockfile-v7.cc | 37 ++-- 9 files changed, 262 insertions(+), 126 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 9624a1c9ebe5..1179d9679434 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -78,13 +78,13 @@ ref call( state.parseExprFromString(callFlakeSchemasNix, state.rootPath(CanonPath::root)), *vCallFlakeSchemas); auto vFlake = state.allocValue(); - flake::callFlake(state, *lockedFlake, *vFlake); + flake::callFlake(state, lockedFlake, *vFlake); auto vDefaultSchemasFlake = state.allocValue(); if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas"))) vDefaultSchemasFlake->mkNull(); else - flake::callFlake(state, *lockedDefaultSchemasFlake, *vDefaultSchemasFlake); + flake::callFlake(state, lockedDefaultSchemasFlake, *vDefaultSchemasFlake); auto vRes = state.allocValue(); Value * args[] = {vDefaultSchemasFlake, vFlake}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 20478296b43e..a14a6b2cf813 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -730,7 +730,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) flake::callFlake( *state, - *flake::lockFlake( + flake::lockFlake( flakeSettings, *state, flakeRef, diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 6c474999f31c..2aa5aa333e48 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -204,7 +204,7 @@ nix_value * nix_locked_flake_get_output_attrs( nix_clear_err(context); try { auto v = nix_alloc_value(context, evalState); - nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value); + nix::flake::callFlake(evalState->state, lockedFlake->lockedFlake.get_ptr(), *v->value); return v; } NIXC_CATCH_ERRS_NULL diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index d1037efcd23b..6b1b061550cf 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -1,79 +1,64 @@ # This is a helper to callFlake() to lazily fetch flake inputs. -# The contents of the lock file, in JSON format. -lockFileStr: +# An external value wrapping the C++ `LockedFlake` object. +lockedFlake: -# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets, -# with sourceInfo.outPath providing an SourceAccessor to a previously -# fetched tree. This is necessary for possibly unlocked inputs, in -# particular the root input, but also --override-inputs pointing to -# unlocked trees. -overrides: +# A primop that, given the locked flake and the input attribute path +# of an input, returns an attribute set mapping the names of its +# inputs to either null (for a regular input) or the input attribute +# path of the target of a "follows" input. +listFlakeInputs: + +# A primop that, given the locked flake and the input attribute path +# of an input, fetches that input and returns an attribute set +# describing it. +fetchFlakeInput: let - inherit (builtins) mapAttrs; - - lockFile = builtins.fromJSON lockFileStr; - - # Resolve a input spec into a node name. An input spec is - # either a node name, or a 'follows' path from the root - # node. - resolveInput = - inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec; - - # Follow an input attrpath (e.g. ["dwarffs" "nixpkgs"]) from the - # root node, returning the final node. - getInputByPath = - nodeName: path: - if path == [ ] then - nodeName - else - getInputByPath - # Since this could be a 'follows' input, call resolveInput. - (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) - (builtins.tail path); - - allNodes = mapAttrs ( - key: node: + inherit (builtins) mapAttrs foldl'; + + # Construct the input denoted by the input attribute path + # `inputAttrPath` (where `[ ]` denotes the top-level flake). This returns `edges` + # (mapping each input name of this input to the input it denotes, + # following "follows" indirections) and `result` (the value of this + # input, i.e. what ends up in the `inputs` attribute of a flake). + mkInput = + inputAttrPath: let - hasOverride = overrides ? ${key}; - isRelative = node.locked.type or null == "path" && builtins.substring 0 1 node.locked.path != "/"; - - parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + info = fetchFlakeInput lockedFlake inputAttrPath; sourceInfo = - if node.buildTime or false then + if info.buildTime then derivation { name = "source"; builder = "builtin:fetch-tree"; system = "builtin"; __structuredAttrs = true; - input = node.locked; + input = info.locked; outputHashMode = "recursive"; - outputHash = node.locked.narHash; + outputHash = info.locked.narHash; } - else if hasOverride then - overrides.${key}.sourceInfo - else if isRelative then - parentNode.sourceInfo else - # FIXME: remove obsolete node.info. - # Note: lock file entries are always final. - builtins.fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]); + info.sourceInfo; - subdir = overrides.${key}.dir or node.locked.dir or ""; + subdir = if info.buildTime then info.locked.dir or "" else info.dir; - outPath = - if !hasOverride && isRelative then - parentNode.outPath + (if node.locked.path == "" then "" else "/" + node.locked.path) - else - sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir); + outPath = sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir); flake = import (outPath + "/flake.nix"); - inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) ( - node.inputs or { } - ); + # Note: constructing `edges` only consults the lock data (via + # `listFlakeInputs`), so it never causes anything to be + # fetched. A regular input is constructed in place; a "follows" + # input is resolved by walking the edges from the top-level + # flake, so every distinct input is constructed (and evaluated) + # only once. + edges = mapAttrs ( + name: target: + if target == null then mkInput (inputAttrPath ++ [ name ]) else getInputByAttrPath target + ) (listFlakeInputs lockedFlake inputAttrPath); + + inputs = mapAttrs (name: input: input.result) edges; outputs = flake.outputs (inputs // { self = result; }); @@ -97,17 +82,22 @@ let in { + inherit edges; + result = - if node.flake or true then + if info.flake then assert builtins.isFunction flake.outputs; - assert !(node.buildTime or false); + assert !info.buildTime; result else sourceInfo // { inherit sourceInfo outPath; }; + }; + + # Follow an input attribute path (e.g. ["dwarffs" "nixpkgs"]) from + # the top-level flake, returning the final input. + getInputByAttrPath = inputAttrPath: foldl' (input: name: input.edges.${name}) root inputAttrPath; - inherit outPath sourceInfo; - } - ) lockFile.nodes; + root = mkInput [ ]; in -allNodes.${lockFile.root}.result +root.result diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 1e8fea69f67f..b8838ab18e52 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -42,7 +42,7 @@ PrimOp getFlake(const Settings & settings) if (args[0]->type() == nPath) { auto path = state.realisePath(pos, *args[0]); - callFlake(state, *lockFlake(settings, state, path, lockFlags), v); + callFlake(state, lockFlake(settings, state, path, lockFlags), v); } else { NixStringContext context; std::string flakeRefS( @@ -73,12 +73,12 @@ PrimOp getFlake(const Settings & settings) auto path = state.storePath(storePath) / CanonPath(subPath); if (!flakeRef.subdir.empty()) path = path / flakeRef.subdir; - return callFlake(state, *lockFlake(settings, state, path, lockFlags), v); + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); } } } - callFlake(state, *lockFlake(settings, state, flakeRef, lockFlags), v); + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); } }; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index b6fa5bd3ab59..939b55dc46f1 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -40,6 +40,8 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" #include "nix/expr/fetch-tree.hh" +#include "nix/expr/json-to-value.hh" +#include "nix/expr/primops.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -595,68 +597,193 @@ static Value * requireInternalFile(EvalState & state, CanonPath path) return v; } -void callFlake(EvalState & state, const LockedFlake & _lockedFlake, Value & vRes) +/** + * An external value wrapping a `LockedFlake`, passed as an argument + * to `call-flake.nix` and consumed by the `listFlakeInputs` and + * `fetchFlakeInput` primops. + */ +class LockedFlakeValue : public ExternalValueBase, public gc_cleanup { - auto & lockedFlake = dynamic_cast(_lockedFlake); +public: + const std::shared_ptr lockedFlake; - auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); + LockedFlakeValue(std::shared_ptr lockedFlake) + : lockedFlake(std::move(lockedFlake)) + { + } - /* Gather the source paths of the nodes that have been fetched - (i.e. the root and the nodes fetched during locking). */ - std::map, SourcePath> nodePaths; + std::string showType() const override + { + return "a locked flake"; + } - nodePaths.emplace(lockedFlake.lockFile.root, lockedFlake.flake.path.parent()); + std::string typeOf() const override + { + return "lockedFlake"; + } - [&](this const auto & recurse, ref node) -> void { - for (auto & [id, input] : node->inputs) { - if (auto child = std::get_if<0>(&input)) { - if (auto sourcePath = *(*child)->sourcePath.lock()) - nodePaths.emplace(*child, *sourcePath); - recurse(*child); - } +protected: + std::ostream & print(std::ostream & str) const override + { + return str << "«locked flake»"; + } +}; + +static const LockedFlake & requireLockedFlake(EvalState & state, Value & v, const PosIdx pos) +{ + state.forceValue(v, pos); + if (v.type() == nExternal) + if (auto * ext = dynamic_cast(v.external())) + return *ext->lockedFlake; + state.error("expected a locked flake but found %1%", showType(v)).atPos(pos).debugThrow(); +} + +static InputAttrPath getInputAttrPathArg(EvalState & state, Value & v, const PosIdx pos) +{ + state.forceList(v, pos, "while evaluating an input attribute path"); + InputAttrPath path; + for (auto elem : v.listView()) + path.push_back( + std::string(state.forceStringNoCtx(*elem, pos, "while evaluating an input attribute path element"))); + return path; +} + +static void prim_listFlakeInputs(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto & lockedFlake = requireLockedFlake(state, *args[0], pos); + auto prefix = getInputAttrPathArg(state, *args[1], pos); + + auto targets = lockedFlake.getInputTargets(prefix); + + auto attrs = state.buildBindings(targets.size()); + + for (auto & [id, target] : targets) { + auto & vTarget = attrs.alloc(state.symbols.create(id)); + if (!target) + vTarget.mkNull(); + else { + auto list = state.buildList(target->size()); + for (const auto & [n, elem] : enumerate(*target)) + (list[n] = state.allocValue())->mkString(elem, state.mem); + vTarget.mkList(list); } - }(lockedFlake.lockFile.root); + } + + v.mkAttrs(attrs); +} - auto overrides = state.buildBindings(nodePaths.size()); +static RegisterPrimOp primop_listFlakeInputs({ + .name = "__listFlakeInputs", + .args = {"lockedFlake", "inputAttrPath"}, + .doc = R"( + For the flake input of *lockedFlake* denoted by *inputAttrPath* + (a list of strings, where the empty list denotes the top-level + flake), return an attribute set mapping the names of its inputs + to either null (for a regular input) or the input attribute path + of the target of a "follows" input. + )", + .impl = prim_listFlakeInputs, + .internal = true, +}); + +static void prim_fetchFlakeInput(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto & lockedFlake = requireLockedFlake(state, *args[0], pos); + auto path = getInputAttrPathArg(state, *args[1], pos); + + std::optional info; + if (!path.empty()) { + info = lockedFlake.findInput(path); + if (!info) + state.error("flake input '%s' does not exist", printInputAttrPath(path)).atPos(pos).debugThrow(); + } - for (auto & [node, sourcePath] : nodePaths) { - auto override = state.buildBindings(2); + auto attrs = state.buildBindings(4); - auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); + attrs.alloc("flake").mkBool(info ? info->isFlake : true); + attrs.alloc("buildTime").mkBool(info && info->buildTime); - auto lockedNode = node.dynamic_pointer_cast(); + if (info && info->buildTime) { + /* Build-time inputs are not fetched at evaluation time; + return the locked input attributes so that call-flake.nix + can construct a `builtin:fetch-tree` derivation. */ + parseJSON(state, fetchers::attrsToJSON(info->lockedRef.toAttrs()).dump(), attrs.alloc("locked")); + } else { + if (info && !info->lockedRef.input.isRelative()) + state.checkURI(info->lockedRef.input.toURLString()); + auto sourcePath = lockedFlake.getSourcePath(state, path); auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs()); + /* Relative path inputs have the same source tree as their + parent flake, so their `sourceInfo` metadata comes from the + nearest non-relative ancestor. */ + auto info2 = info; + while (info2 && info2->lockedRef.input.isRelative()) { + assert(info2->parentInputAttrPath); + if (info2->parentInputAttrPath->empty()) + /* The parent is the top-level flake. */ + info2.reset(); + else + info2 = lockedFlake.findInput(*info2->parentInputAttrPath); + } + emitTreeAttrs( state, storePath, - lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input, - vSourceInfo, + info2 ? info2->lockedRef.input : lockedFlake.flake.lockedRef.input, + attrs.alloc("sourceInfo"), false, - !lockedNode && lockedFlake.flake.forceDirty); - - auto key = keyMap.find(node); - assert(key != keyMap.end()); + !info2 && lockedFlake.flake.forceDirty); - override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem); - - overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); + attrs.alloc("dir").mkString(CanonPath(subdir).rel(), state.mem); } - auto & vOverrides = state.allocValue()->mkAttrs(overrides); + v.mkAttrs(attrs); +} - Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); +static RegisterPrimOp primop_fetchFlakeInput({ + .name = "__fetchFlakeInput", + .args = {"lockedFlake", "inputAttrPath"}, + .doc = R"( + Fetch the flake input of *lockedFlake* denoted by *inputAttrPath* + (a list of strings, where the empty list denotes the top-level + flake) and return an attribute set describing it: `flake` + (whether it's a flake), `buildTime` (whether it's fetched at + build time), and either `locked` (the locked input attributes, + for build-time inputs, which are not fetched) or `sourceInfo` + (the fetched tree's metadata) and `dir` (the subdirectory of the + flake within `sourceInfo`). + )", + .impl = prim_fetchFlakeInput, + .internal = true, +}); + +void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & vRes) +{ + auto vLockedFlake = state.allocValue(); + vLockedFlake->mkExternal(new LockedFlakeValue(std::move(lockedFlake))); - auto vLocks = state.allocValue(); - vLocks->mkString(lockFileStr, state.mem); + Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); - Value * args[] = {vLocks, &vOverrides}; + Value * args[] = { + vLockedFlake, + **get(state.internalPrimOps, "listFlakeInputs"), + **get(state.internalPrimOps, "fetchFlakeInput"), + }; state.callFunction(*vCallFlake, args, vRes, noPos); } LockedFlake::~LockedFlake() {} +std::vector LockedFlake::getInputNames(const InputAttrPath & prefix) const +{ + std::vector res; + for (auto & [name, target] : getInputTargets(prefix)) + res.push_back(name); + return res; +} + std::string LockedFlake::to_string() const { return toJSON().dump(2); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index ac21388d652e..06695eed9786 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -157,11 +157,23 @@ struct LockedFlake virtual ~LockedFlake(); + /** + * For the input denoted by `prefix` (or the top-level flake if + * `prefix` is empty), return a map from the names of its inputs + * to the target of that input: for a regular input, std::nullopt; + * for a "follows" input, the input attribute path (relative to + * the top-level flake) of the immediate target of the + * "follows". Note that the target may itself denote a "follows" + * input. Throws an error if `prefix` does not denote an existing + * input. + */ + virtual std::map> getInputTargets(const InputAttrPath & prefix) const = 0; + /** * Return the names of the inputs of the input denoted by * `prefix`, or of the top-level flake if `prefix` is empty. */ - virtual std::vector getInputNames(const InputAttrPath & prefix) const = 0; + std::vector getInputNames(const InputAttrPath & prefix) const; /** * Information about a locked input. @@ -171,6 +183,12 @@ struct LockedFlake FlakeRef lockedRef; bool isFlake = true; bool buildTime = false; + + /** + * For relative path inputs, the input attribute path of the + * flake relative to which the path is interpreted. + */ + std::optional parentInputAttrPath; }; /** @@ -337,7 +355,7 @@ std::unique_ptr lockFlake( std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); -void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); +void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & v); } // namespace flake diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index 13ad062c5550..78e661dcfd09 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -79,11 +79,7 @@ struct LockFileV7 LockFileV7() {}; LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path); - typedef std::map, std::string> KeyMap; - - std::pair toJSON() const; - - std::pair to_string() const; + nlohmann::json toJSON() const; /** * Check whether this lock file has any unlocked or non-final @@ -120,7 +116,7 @@ struct LockedFlakeV7 : LockedFlake LockedFlakeV7( const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); - std::vector getInputNames(const InputAttrPath & prefix) const override; + std::map> getInputTargets(const InputAttrPath & prefix) const override; std::optional findInput(const InputAttrPath & path) const override; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 99ad7c4a5168..91f93486abfc 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -182,10 +182,10 @@ LockFileV7::LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann: // a bit since we don't need to worry about cycles. } -std::pair LockFileV7::toJSON() const +nlohmann::json LockFileV7::toJSON() const { nlohmann::json nodes; - KeyMap nodeKeys; + std::map, std::string> nodeKeys; boost::unordered_flat_set keys; auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { @@ -244,13 +244,7 @@ std::pair LockFileV7::toJSON() const json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); - return {json, std::move(nodeKeys)}; -} - -std::pair LockFileV7::to_string() const -{ - auto [json, nodeKeys] = toJSON(); - return {json.dump(2), std::move(nodeKeys)}; + return json; } std::optional LockFileV7::isUnlocked(const fetchers::Settings & fetchSettings) const @@ -288,7 +282,7 @@ std::optional LockFileV7::isUnlocked(const fetchers::Settings & fetchS bool LockFileV7::operator==(const LockFileV7 & other) const { // FIXME: slow - return toJSON().first == other.toJSON().first; + return toJSON() == other.toJSON(); } std::map LockFileV7::getAllInputs() const @@ -395,12 +389,21 @@ void LockFileV7::check() } } -std::vector LockedFlakeV7::getInputNames(const InputAttrPath & prefix) const +std::map> LockedFlakeV7::getInputTargets(const InputAttrPath & prefix) const { - std::vector res; - if (auto node = lockFile.findInput(prefix)) - for (auto & [id, input] : node->inputs) - res.push_back(id); + auto node = lockFile.findInput(prefix); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); + + std::map> res; + + for (auto & [id, input] : node->inputs) { + if (std::get_if<0>(&input)) + res.emplace(id, std::nullopt); + else + res.emplace(id, std::get<1>(input)); + } + return res; } @@ -410,6 +413,8 @@ std::optional LockedFlakeV7::findInput(const InputAttrPa return InputInfo{ .lockedRef = node->lockedRef, .isFlake = node->isFlake, + .buildTime = node->buildTime, + .parentInputAttrPath = node->parentInputAttrPath, }; return std::nullopt; } @@ -492,7 +497,7 @@ std::optional LockedFlakeV7::isUnlocked(const fetchers::Settings & fet nlohmann::json LockedFlakeV7::toJSON() const { - return lockFile.toJSON().first; + return lockFile.toJSON(); } LockedFlakeV7::LockedFlakeV7( From 16335edc47246a5ab8c809f3b90b2379a66c8beb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2026 16:43:19 +0200 Subject: [PATCH 08/26] Move Node/LockedNode/LockFileV7/LockedFlakeV7 into lockfile-v7.cc These types are no longer used outside of lockfile-v7.cc, so they can be implementation details. The header now only exposes two free functions returning `std::unique_ptr`: * `parseLockFileV7()`: construct a `LockedFlake` from the JSON contents of a version 5-7 lock file. * `lockFlakeV7()` (previously `LockedFlakeV7::lockFlake()`): compute a new lock file. This also keeps the version dispatch in the free function `lockFlake()` symmetrical for the future version 8 implementation: it will just pick between `parseLockFileV{7,8}()` and `lockFlakeV{7,8}()` based on the `version` field in the lock file JSON. Also remove the unused `LockedNode::computeStorePath()`. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 10 +- src/libflake/include/nix/flake/lockfile-v7.hh | 143 ++--------------- src/libflake/lockfile-v7.cc | 144 ++++++++++++++++-- 3 files changed, 149 insertions(+), 148 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 939b55dc46f1..f0c45ccd8e36 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -453,11 +453,11 @@ std::unique_ptr lockFlake( } // FIXME: dispatch on the lock file version here. - LockedFlakeV7 oldLockedFlake(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); + auto oldLockedFlake = parseLockFileV7(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); - debug("old lock file: %s", oldLockedFlake.to_string()); + debug("old lock file: %s", oldLockedFlake->to_string()); - auto lockedFlake = LockedFlakeV7::lockFlake(settings, state, lockFlags, std::move(flake), oldLockedFlake); + auto lockedFlake = lockFlakeV7(settings, state, lockFlags, std::move(flake), *oldLockedFlake); debug("new lock file: %s", lockedFlake->to_string()); @@ -465,9 +465,9 @@ std::unique_ptr lockFlake( /* Check whether we need to / can write the new lock file. */ auto lockedFlakeJson = lockedFlake->toJSON(); - if (lockedFlakeJson != oldLockedFlake.toJSON() || lockFlags.outputLockFilePath) { + if (lockedFlakeJson != oldLockedFlake->toJSON() || lockFlags.outputLockFilePath) { - auto diff = lockedFlake->diff(oldLockedFlake); + auto diff = lockedFlake->diff(*oldLockedFlake); if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh index 78e661dcfd09..ce86696abfce 100644 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ b/src/libflake/include/nix/flake/lockfile-v7.hh @@ -2,145 +2,28 @@ ///@file #include "nix/flake/flake.hh" -#include "nix/util/sync.hh" #include -namespace nix { -class Store; -class StorePath; -} // namespace nix - namespace nix::flake { -struct LockedNode; - /** - * A node in the lock file. It has outgoing edges to other nodes (its - * inputs). Only the root node has this type; all other nodes have - * type LockedNode. + * Parse a lock file in the old graph-based format (versions 5-7). + * `json` must be null if the lock file doesn't exist. */ -struct Node : std::enable_shared_from_this -{ - typedef std::variant, InputAttrPath> Edge; - - std::map inputs; - - virtual ~Node() {} -}; +std::unique_ptr parseLockFileV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); /** - * A non-root node in the lock file. + * Compute a version 7 lock file for `flake`, reusing entries from + * `oldLockFile` (which must have been produced by `parseLockFileV7()`) + * where possible. Note: this does not write the new lock file. */ -struct LockedNode : Node -{ - FlakeRef lockedRef, originalRef; - bool isFlake = true; - bool buildTime = false; - - /* The node relative to which relative source paths - (e.g. 'path:../foo') are interpreted. */ - std::optional parentInputAttrPath; - - /** - * The source path of this node, if it has been fetched. Set by - * `LockedFlakeV7::lockFlake()` for nodes fetched during locking, - * and by `LockedFlakeV7::getSourcePath()` for nodes fetched on - * demand. - */ - mutable Sync> sourcePath; - - LockedNode( - const FlakeRef & lockedRef, - const FlakeRef & originalRef, - bool isFlake = true, - bool buildTime = false, - std::optional parentInputAttrPath = {}) - : lockedRef(std::move(lockedRef)) - , originalRef(std::move(originalRef)) - , isFlake(isFlake) - , buildTime(buildTime) - , parentInputAttrPath(std::move(parentInputAttrPath)) - { - } - - LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); - - StorePath computeStorePath(Store & store) const; -}; - -/** - * The old graph-based lock file format (versions 5-7). - */ -struct LockFileV7 -{ - ref root = make_ref(); - - LockFileV7() {}; - LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path); - - nlohmann::json toJSON() const; - - /** - * Check whether this lock file has any unlocked or non-final - * inputs. If so, return one. - */ - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; - - bool operator==(const LockFileV7 & other) const; - - std::shared_ptr findInput(const InputAttrPath & path) const; - - std::map getAllInputs() const; - - /** - * Check that every 'follows' input target exists. - */ - void check(); -}; - -struct LockedFlakeV7 : LockedFlake -{ - LockFileV7 lockFile; - - LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) - : LockedFlake(std::move(flake)) - , lockFile(std::move(lockFile)) - { - } - - /** - * Construct from the JSON contents of a lock file (which must be - * null if the lock file doesn't exist). - */ - LockedFlakeV7( - const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); - - std::map> getInputTargets(const InputAttrPath & prefix) const override; - - std::optional findInput(const InputAttrPath & path) const override; - - SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override; - - void visit(VisitCallback callback) const override; - - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; - - std::string diff(const LockedFlake & oldLockFile) const override; - - nlohmann::json toJSON() const override; - - /** - * Compute a lock file for `flake`, reusing entries from - * `oldLockFile` (which must be a `LockedFlakeV7`) where - * possible. Note: this does not write the new lock file. - */ - static std::unique_ptr lockFlake( - const Settings & settings, - EvalState & state, - const LockFlags & lockFlags, - Flake flake, - const LockedFlake & oldLockFile); -}; +std::unique_ptr lockFlakeV7( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); } // namespace nix::flake diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 91f93486abfc..a704118e698c 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -22,6 +22,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/lockfile-v7.hh" +#include "nix/util/sync.hh" #include "nix/flake/settings.hh" #include "nix/expr/eval.hh" #include "nix/expr/nixexpr.hh" @@ -49,6 +50,129 @@ class Store; namespace nix::flake { +struct LockedNode; + +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ +struct Node : std::enable_shared_from_this +{ + typedef std::variant, InputAttrPath> Edge; + + std::map inputs; + + virtual ~Node() {} +}; + +/** + * A non-root node in the lock file. + */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + bool buildTime = false; + + /* The node relative to which relative source paths + (e.g. 'path:../foo') are interpreted. */ + std::optional parentInputAttrPath; + + /** + * The source path of this node, if it has been fetched. Set by + * `LockedFlakeV7::lockFlake()` for nodes fetched during locking, + * and by `LockedFlakeV7::getSourcePath()` for nodes fetched on + * demand. + */ + mutable Sync> sourcePath; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true, + bool buildTime = false, + std::optional parentInputAttrPath = {}) + : lockedRef(std::move(lockedRef)) + , originalRef(std::move(originalRef)) + , isFlake(isFlake) + , buildTime(buildTime) + , parentInputAttrPath(std::move(parentInputAttrPath)) + { + } + + LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); +}; + +/** + * The old graph-based lock file format (versions 5-7). + */ +struct LockFileV7 +{ + ref root = make_ref(); + + LockFileV7() {}; + LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path); + + nlohmann::json toJSON() const; + + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; + + bool operator==(const LockFileV7 & other) const; + + std::shared_ptr findInput(const InputAttrPath & path) const; + + std::map getAllInputs() const; + + /** + * Check that every 'follows' input target exists. + */ + void check(); +}; + +struct LockedFlakeV7 : LockedFlake +{ + /** + * The lock file in the old graph-based format (versions 5-7). + */ + LockFileV7 lockFile; + + LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + { + } + + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) + { + } + + std::map> getInputTargets(const InputAttrPath & prefix) const override; + + std::optional findInput(const InputAttrPath & path) const override; + + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override; + + void visit(VisitCallback callback) const override; + + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; + + std::string diff(const LockedFlake & oldLockFile) const override; + + nlohmann::json toJSON() const override; +}; + static FlakeRef getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) { @@ -94,11 +218,6 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann: lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); } -StorePath LockedNode::computeStorePath(Store & store) const -{ - return lockedRef.input.computeStorePath(store); -} - static std::shared_ptr doFind(const ref & root, const InputAttrPath & path, std::vector & visited) { @@ -500,13 +619,6 @@ nlohmann::json LockedFlakeV7::toJSON() const return lockFile.toJSON(); } -LockedFlakeV7::LockedFlakeV7( - const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) - : LockedFlake(std::move(flake)) - , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) -{ -} - static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { if (!lockFilePath.pathExists()) @@ -523,7 +635,13 @@ static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const S return LockFileV7(fetchSettings, json, fmt("%s", lockFilePath)); } -std::unique_ptr LockedFlakeV7::lockFlake( +std::unique_ptr parseLockFileV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) +{ + return std::make_unique(fetchSettings, std::move(flake), json, path); +} + +std::unique_ptr lockFlakeV7( const Settings & settings, EvalState & state, const LockFlags & lockFlags, From 472b0f5e21e82db13d0f4d0c3c92fb32e6c5ccc4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2026 16:58:01 +0200 Subject: [PATCH 09/26] lockfile-v7.cc: Move method definitions into the classes Now that these classes are local to this file, there is no need for separate declarations and definitions. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 1 - src/libflake/include/nix/flake/flake.hh | 19 + src/libflake/include/nix/flake/lockfile-v7.hh | 29 - src/libflake/include/nix/flake/meson.build | 1 - src/libflake/lockfile-v7.cc | 766 +++++++++--------- 5 files changed, 389 insertions(+), 427 deletions(-) delete mode 100644 src/libflake/include/nix/flake/lockfile-v7.hh diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index f0c45ccd8e36..1f84c50dda98 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -25,7 +25,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-cache.hh" #include "nix/expr/eval-settings.hh" -#include "nix/flake/lockfile-v7.hh" #include "nix/expr/eval-inline.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 06695eed9786..72894f3d7312 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -355,6 +355,25 @@ std::unique_ptr lockFlake( std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); +/** + * Parse a lock file in the old graph-based format (versions 5-7). + * `json` must be null if the lock file doesn't exist. + */ +std::unique_ptr parseLockFileV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); + +/** + * Compute a version 7 lock file for `flake`, reusing entries from + * `oldLockFile` (which must have been produced by `parseLockFileV7()`) + * where possible. Note: this does not write the new lock file. + */ +std::unique_ptr lockFlakeV7( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); + void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & v); } // namespace flake diff --git a/src/libflake/include/nix/flake/lockfile-v7.hh b/src/libflake/include/nix/flake/lockfile-v7.hh deleted file mode 100644 index ce86696abfce..000000000000 --- a/src/libflake/include/nix/flake/lockfile-v7.hh +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -///@file - -#include "nix/flake/flake.hh" - -#include - -namespace nix::flake { - -/** - * Parse a lock file in the old graph-based format (versions 5-7). - * `json` must be null if the lock file doesn't exist. - */ -std::unique_ptr parseLockFileV7( - const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); - -/** - * Compute a version 7 lock file for `flake`, reusing entries from - * `oldLockFile` (which must have been produced by `parseLockFileV7()`) - * where possible. Note: this does not write the new lock file. - */ -std::unique_ptr lockFlakeV7( - const Settings & settings, - EvalState & state, - const LockFlags & lockFlags, - Flake flake, - const LockedFlake & oldLockFile); - -} // namespace nix::flake diff --git a/src/libflake/include/nix/flake/meson.build b/src/libflake/include/nix/flake/meson.build index 404ee22a3cf5..dbe66ef229b3 100644 --- a/src/libflake/include/nix/flake/meson.build +++ b/src/libflake/include/nix/flake/meson.build @@ -6,7 +6,6 @@ headers = files( 'flake.hh', 'flakeref.hh', 'input-attr-path.hh', - 'lockfile-v7.hh', 'provenance.hh', 'settings.hh', 'url-name.hh', diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index a704118e698c..ca64a007648a 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -21,7 +21,7 @@ #include #include "nix/fetchers/fetch-settings.hh" -#include "nix/flake/lockfile-v7.hh" +#include "nix/flake/flake.hh" #include "nix/util/sync.hh" #include "nix/flake/settings.hh" #include "nix/expr/eval.hh" @@ -50,6 +50,26 @@ class Store; namespace nix::flake { +static FlakeRef +getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = fetchers::jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : fetchers::jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(fetchSettings, attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + struct LockedNode; /** @@ -81,8 +101,8 @@ struct LockedNode : Node /** * The source path of this node, if it has been fetched. Set by - * `LockedFlakeV7::lockFlake()` for nodes fetched during locking, - * and by `LockedFlakeV7::getSourcePath()` for nodes fetched on + * `lockFlakeV7()` for nodes fetched during locking, and by + * `LockedFlakeV7::getSourcePath()` for nodes fetched on * demand. */ mutable Sync> sourcePath; @@ -101,122 +121,31 @@ struct LockedNode : Node { } - LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); -}; - -/** - * The old graph-based lock file format (versions 5-7). - */ -struct LockFileV7 -{ - ref root = make_ref(); - - LockFileV7() {}; - LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path); - - nlohmann::json toJSON() const; - - /** - * Check whether this lock file has any unlocked or non-final - * inputs. If so, return one. - */ - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; - - bool operator==(const LockFileV7 & other) const; - - std::shared_ptr findInput(const InputAttrPath & path) const; - - std::map getAllInputs() const; - - /** - * Check that every 'follows' input target exists. - */ - void check(); -}; - -struct LockedFlakeV7 : LockedFlake -{ - /** - * The lock file in the old graph-based format (versions 5-7). - */ - LockFileV7 lockFile; - - LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) - : LockedFlake(std::move(flake)) - , lockFile(std::move(lockFile)) - { - } - - /** - * Construct from the JSON contents of a lock file (which must be - * null if the lock file doesn't exist). - */ - LockedFlakeV7( - const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) - : LockedFlake(std::move(flake)) - , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) + LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) + , parentInputAttrPath( + json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { - } - - std::map> getInputTargets(const InputAttrPath & prefix) const override; - - std::optional findInput(const InputAttrPath & path) const override; - - SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override; - - void visit(VisitCallback callback) const override; - - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override; - - std::string diff(const LockedFlake & oldLockFile) const override; - - nlohmann::json toJSON() const override; -}; - -static FlakeRef -getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) -{ - auto i = json.find(attr); - if (i != json.end()) { - auto attrs = fetchers::jsonToAttrs(*i); - // FIXME: remove when we drop support for version 5. - if (info) { - auto j = json.find(info); - if (j != json.end()) { - for (auto k : fetchers::jsonToAttrs(*j)) - attrs.insert_or_assign(k.first, k.second); - } + if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { + if (lockedRef.input.getNarHash()) + warn( + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", + lockedRef.to_string()); + else + throw Error( + "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); } - return FlakeRef::fromAttrs(fetchSettings, attrs); - } - - throw Error("attribute '%s' missing in lock file", attr); -} -LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) - : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) - , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) - , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) - , parentInputAttrPath( - json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) -{ - if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { - if (lockedRef.input.getNarHash()) - warn( - "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " - "This is not reproducible and will break after garbage collection or when shared.", - lockedRef.to_string()); - else - throw Error( - "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", - fetchers::attrsToJSON(lockedRef.input.toAttrs())); + // For backward compatibility, lock file entries are implicitly final. + assert(!lockedRef.input.attrs.contains("__final")); + lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); } - - // For backward compatibility, lock file entries are implicitly final. - assert(!lockedRef.input.attrs.contains("__final")); - lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); -} +}; static std::shared_ptr doFind(const ref & root, const InputAttrPath & path, std::vector & visited) @@ -250,180 +179,212 @@ doFind(const ref & root, const InputAttrPath & path, std::vector LockFileV7::findInput(const InputAttrPath & path) const -{ - std::vector visited; - return doFind(root, path, visited); -} - -LockFileV7::LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) +/** + * The old graph-based lock file format (versions 5-7). + */ +struct LockFileV7 { - auto version = json.value("version", 0); - if (version < 5 || version > 7) - throw Error("lock file '%s' has unsupported version %d", path, version); + ref root = make_ref(); - std::string rootKey = json["root"]; - std::map> nodeMap{{rootKey, root}}; + LockFileV7() {}; - [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { - if (jsonNode.find("inputs") == jsonNode.end()) - return; - for (auto & i : jsonNode["inputs"].items()) { - if (i.value().is_array()) { // FIXME: remove, obsolete - InputAttrPath path; - for (auto & j : i.value()) - path.push_back(j); - node.inputs.insert_or_assign(i.key(), path); - } else { - std::string inputKey = i.value(); - auto k = nodeMap.find(inputKey); - if (k == nodeMap.end()) { - auto & nodes = json["nodes"]; - auto jsonNode2 = nodes.find(inputKey); - if (jsonNode2 == nodes.end()) - throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(fetchSettings, *jsonNode2); - k = nodeMap.insert_or_assign(inputKey, input).first; - getInputs(*input, *jsonNode2); + LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) + { + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + std::string rootKey = json["root"]; + std::map> nodeMap{{rootKey, root}}; + + [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { + if (jsonNode.find("inputs") == jsonNode.end()) + return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { // FIXME: remove, obsolete + InputAttrPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto & nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = make_ref(fetchSettings, *jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, *jsonNode2); + } + if (auto child = k->second.dynamic_pointer_cast()) + node.inputs.insert_or_assign(i.key(), ref(child)); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); } - if (auto child = k->second.dynamic_pointer_cast()) - node.inputs.insert_or_assign(i.key(), ref(child)); - else - // FIXME: replace by follows node - throw Error("lock file contains cycle to root node"); } - } - }(*root, json["nodes"][rootKey]); + }(*root, json["nodes"][rootKey]); - // FIXME: check that there are no cycles in version >= 7. Cycles - // between inputs are only possible using 'follows' indirections. - // Once we drop support for version <= 6, we can simplify the code - // a bit since we don't need to worry about cycles. -} + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. + } -nlohmann::json LockFileV7::toJSON() const -{ - nlohmann::json nodes; - std::map, std::string> nodeKeys; - boost::unordered_flat_set keys; - - auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { - auto k = nodeKeys.find(node); - if (k != nodeKeys.end()) - return k->second; - - if (!keys.insert(key).second) { - for (int n = 2;; ++n) { - auto k = fmt("%s_%d", key, n); - if (keys.insert(k).second) { - key = k; - break; + nlohmann::json toJSON() const + { + nlohmann::json nodes; + std::map, std::string> nodeKeys; + boost::unordered_flat_set keys; + + auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2;; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } } } - } - nodeKeys.insert_or_assign(node, key); + nodeKeys.insert_or_assign(node, key); - auto n = nlohmann::json::object(); + auto n = nlohmann::json::object(); - if (!node->inputs.empty()) { - auto inputs = nlohmann::json::object(); - for (auto & i : node->inputs) { - if (auto child = std::get_if<0>(&i.second)) { - inputs[i.first] = dumpNode(i.first, *child); - } else if (auto follows = std::get_if<1>(&i.second)) { - auto arr = nlohmann::json::array(); - for (auto & x : *follows) - arr.push_back(x); - inputs[i.first] = std::move(arr); + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } } + n["inputs"] = std::move(inputs); } - n["inputs"] = std::move(inputs); - } - if (auto lockedNode = node.dynamic_pointer_cast()) { - n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); - n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); - if (!lockedNode->isFlake) - n["flake"] = false; - if (lockedNode->buildTime) - n["buildTime"] = true; - if (lockedNode->parentInputAttrPath) - n["parent"] = *lockedNode->parentInputAttrPath; - } + if (auto lockedNode = node.dynamic_pointer_cast()) { + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); + assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); + if (!lockedNode->isFlake) + n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; + if (lockedNode->parentInputAttrPath) + n["parent"] = *lockedNode->parentInputAttrPath; + } - nodes[key] = std::move(n); + nodes[key] = std::move(n); - return key; - }; + return key; + }; - nlohmann::json json; - json["version"] = 7; - json["root"] = dumpNode("root", root); - json["nodes"] = std::move(nodes); + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); - return json; -} + return json; + } -std::optional LockFileV7::isUnlocked(const fetchers::Settings & fetchSettings) const -{ - std::set> nodes; + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const + { + std::set> nodes; + + [&](this const auto & visit, ref node) { + if (!nodes.insert(node).second) + return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }(root); + + /* Return whether the input is either locked, or, if + `allow-dirty-locks` is enabled, it has a NAR hash. In the + latter case, we can verify the input but we may not be able to + fetch it from anywhere. */ + auto isConsideredLocked = [&](const fetchers::Input & input) { + return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); + }; - [&](this const auto & visit, ref node) { - if (!nodes.insert(node).second) - return; - for (auto & i : node->inputs) - if (auto child = std::get_if<0>(&i.second)) - visit(*child); - }(root); - - /* Return whether the input is either locked, or, if - `allow-dirty-locks` is enabled, it has a NAR hash. In the - latter case, we can verify the input but we may not be able to - fetch it from anywhere. */ - auto isConsideredLocked = [&](const fetchers::Input & input) { - return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); - }; + for (auto & i : nodes) { + if (i == ref(root)) + continue; + auto node = i.dynamic_pointer_cast(); + if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) + && !node->lockedRef.input.isRelative()) + return node->lockedRef; + } - for (auto & i : nodes) { - if (i == ref(root)) - continue; - auto node = i.dynamic_pointer_cast(); - if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) - && !node->lockedRef.input.isRelative()) - return node->lockedRef; + return {}; } - return {}; -} + bool operator==(const LockFileV7 & other) const + { + // FIXME: slow + return toJSON() == other.toJSON(); + } -bool LockFileV7::operator==(const LockFileV7 & other) const -{ - // FIXME: slow - return toJSON() == other.toJSON(); -} + std::shared_ptr findInput(const InputAttrPath & path) const + { + std::vector visited; + return doFind(root, path, visited); + } -std::map LockFileV7::getAllInputs() const -{ - std::set> done; - std::map res; + std::map getAllInputs() const + { + std::set> done; + std::map res; + + [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { + if (!done.insert(node).second) + return; + + for (auto & [id, input] : node->inputs) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + res.emplace(inputAttrPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputAttrPath, *child); + } + }({}, root); - [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { - if (!done.insert(node).second) - return; + return res; + } - for (auto & [id, input] : node->inputs) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(id); - res.emplace(inputAttrPath, input); - if (auto child = std::get_if<0>(&input)) - recurse(inputAttrPath, *child); + /** + * Check that every 'follows' input target exists. + */ + void check() + { + auto inputs = getAllInputs(); + + for (auto & [inputAttrPath, input] : inputs) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !findInput(*follows)) + throw Error( + "input '%s' follows a non-existent input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(*follows)); + } } - }({}, root); - - return res; -} + } +}; static std::string describe(const FlakeRef & flakeRef) { @@ -455,169 +416,182 @@ static bool equals(const Node::Edge & e1, const Node::Edge & e2) return false; } -std::string LockedFlakeV7::diff(const LockedFlake & _oldLockFile) const +struct LockedFlakeV7 : LockedFlake { - /* If `oldLockFile` is not a version 7 lock file, diff against an - empty lock file, i.e. all inputs of this lock file will show up - as added. */ - auto oldLockFile = dynamic_cast(&_oldLockFile); - - auto oldFlat = oldLockFile ? oldLockFile->lockFile.getAllInputs() : std::map(); - auto newFlat = lockFile.getAllInputs(); - - auto i = oldFlat.begin(); - auto j = newFlat.begin(); - std::string res; - - while (i != oldFlat.end() || j != newFlat.end()) { - if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { - res += fmt( - "• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", printInputAttrPath(j->first), j->second); - ++j; - } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { - res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); - ++i; - } else { - if (!equals(i->second, j->second)) { - res += - fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", - printInputAttrPath(i->first), - i->second, - j->second); - } - ++i; - ++j; - } + /** + * The lock file in the old graph-based format (versions 5-7). + */ + LockFileV7 lockFile; + + LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + { } - return res; -} + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) + { + } -void LockFileV7::check() -{ - auto inputs = getAllInputs(); + std::map> getInputTargets(const InputAttrPath & prefix) const override + { + auto node = lockFile.findInput(prefix); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); - for (auto & [inputAttrPath, input] : inputs) { - if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !findInput(*follows)) - throw Error( - "input '%s' follows a non-existent input '%s'", - printInputAttrPath(inputAttrPath), - printInputAttrPath(*follows)); + std::map> res; + + for (auto & [id, input] : node->inputs) { + if (std::get_if<0>(&input)) + res.emplace(id, std::nullopt); + else + res.emplace(id, std::get<1>(input)); } + + return res; } -} -std::map> LockedFlakeV7::getInputTargets(const InputAttrPath & prefix) const -{ - auto node = lockFile.findInput(prefix); - if (!node) - throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); + std::optional findInput(const InputAttrPath & path) const override + { + if (auto node = std::dynamic_pointer_cast(lockFile.findInput(path))) + return InputInfo{ + .lockedRef = node->lockedRef, + .isFlake = node->isFlake, + .buildTime = node->buildTime, + .parentInputAttrPath = node->parentInputAttrPath, + }; + return std::nullopt; + } - std::map> res; + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override + { + /* The root node. */ + if (inputAttrPath.empty()) + return flake.path.parent(); - for (auto & [id, input] : node->inputs) { - if (std::get_if<0>(&input)) - res.emplace(id, std::nullopt); - else - res.emplace(id, std::get<1>(input)); - } + auto node = lockFile.findInput(inputAttrPath); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); - return res; -} + auto lockedNode = std::dynamic_pointer_cast(node); + assert(lockedNode); -std::optional LockedFlakeV7::findInput(const InputAttrPath & path) const -{ - if (auto node = std::dynamic_pointer_cast(lockFile.findInput(path))) - return InputInfo{ - .lockedRef = node->lockedRef, - .isFlake = node->isFlake, - .buildTime = node->buildTime, - .parentInputAttrPath = node->parentInputAttrPath, - }; - return std::nullopt; -} + { + auto sourcePath(lockedNode->sourcePath.lock()); + if (*sourcePath) + return **sourcePath; + } -SourcePath LockedFlakeV7::getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const -{ - /* The root node. */ - if (inputAttrPath.empty()) - return flake.path.parent(); + /* Note: we fetch without holding the `sourcePath` lock, so + concurrent calls don't get serialized. Racing fetches of the + same node are harmless since they produce the same path. */ + auto path = [&]() -> SourcePath { + if (auto relativePath = lockedNode->lockedRef.input.isRelative()) { + /* Resolve relative path inputs against the source path of + their parent flake. */ + auto parentPath = getSourcePath(state, lockedNode->parentInputAttrPath.value()); + return {parentPath.accessor, CanonPath(relativePath->string(), parentPath.path)}; + } else { + /* Note: `lockedRef` is a copy since `mountInput()` may + modify the input (e.g. adding a `narHash` attribute). */ + auto lockedRef = lockedNode->lockedRef; + auto accessor = + state.inputCache + ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) + .accessor; + return state.storePath( + state.mountInput(lockedRef.input, lockedNode->lockedRef.input, accessor, true, true)) + / CanonPath(lockedRef.subdir); + } + }(); - auto node = lockFile.findInput(inputAttrPath); - if (!node) - throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + *lockedNode->sourcePath.lock() = path; - auto lockedNode = std::dynamic_pointer_cast(node); - assert(lockedNode); + return path; + } + void visit(VisitCallback callback) const override { - auto sourcePath(lockedNode->sourcePath.lock()); - if (*sourcePath) - return **sourcePath; + if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) + return; + + [&](this const auto & recurse, const InputAttrPath & prefix, ref node) -> void { + for (auto & [id, input] : node->inputs) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + if (auto child = std::get_if<0>(&input)) { + if (callback( + inputAttrPath, + InputInfo{ + .lockedRef = (*child)->lockedRef, + .isFlake = (*child)->isFlake, + .buildTime = (*child)->buildTime, + })) + recurse(inputAttrPath, *child); + } else if (auto follows = std::get_if<1>(&input)) { + callback(inputAttrPath, *follows); + } + } + }({}, lockFile.root); } - /* Note: we fetch without holding the `sourcePath` lock, so - concurrent calls don't get serialized. Racing fetches of the - same node are harmless since they produce the same path. */ - auto path = [&]() -> SourcePath { - if (auto relativePath = lockedNode->lockedRef.input.isRelative()) { - /* Resolve relative path inputs against the source path of - their parent flake. */ - auto parentPath = getSourcePath(state, lockedNode->parentInputAttrPath.value()); - return {parentPath.accessor, CanonPath(relativePath->string(), parentPath.path)}; - } else { - /* Note: `lockedRef` is a copy since `mountInput()` may - modify the input (e.g. adding a `narHash` attribute). */ - auto lockedRef = lockedNode->lockedRef; - auto accessor = - state.inputCache - ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) - .accessor; - return state.storePath(state.mountInput(lockedRef.input, lockedNode->lockedRef.input, accessor, true, true)) - / CanonPath(lockedRef.subdir); - } - }(); + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override + { + return lockFile.isUnlocked(fetchSettings); + } - *lockedNode->sourcePath.lock() = path; + std::string diff(const LockedFlake & _oldLockFile) const override + { + /* If `oldLockFile` is not a version 7 lock file, diff against an + empty lock file, i.e. all inputs of this lock file will show up + as added. */ + auto oldLockFile = dynamic_cast(&_oldLockFile); - return path; -} + auto oldFlat = oldLockFile ? oldLockFile->lockFile.getAllInputs() : std::map(); + auto newFlat = lockFile.getAllInputs(); -void LockedFlakeV7::visit(VisitCallback callback) const -{ - if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) - return; + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; - [&](this const auto & recurse, const InputAttrPath & prefix, ref node) -> void { - for (auto & [id, input] : node->inputs) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(id); - if (auto child = std::get_if<0>(&input)) { - if (callback( - inputAttrPath, - InputInfo{ - .lockedRef = (*child)->lockedRef, - .isFlake = (*child)->isFlake, - .buildTime = (*child)->buildTime, - })) - recurse(inputAttrPath, *child); - } else if (auto follows = std::get_if<1>(&input)) { - callback(inputAttrPath, *follows); + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += + fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", + printInputAttrPath(j->first), + j->second); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); + ++i; + } else { + if (!equals(i->second, j->second)) { + res += + fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", + printInputAttrPath(i->first), + i->second, + j->second); + } + ++i; + ++j; } } - }({}, lockFile.root); -} -std::optional LockedFlakeV7::isUnlocked(const fetchers::Settings & fetchSettings) const -{ - return lockFile.isUnlocked(fetchSettings); -} + return res; + } -nlohmann::json LockedFlakeV7::toJSON() const -{ - return lockFile.toJSON(); -} + nlohmann::json toJSON() const override + { + return lockFile.toJSON(); + } +}; static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { From eb49bf0234adaf83f05259aa0f1342551d698d54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2026 18:57:22 +0200 Subject: [PATCH 10/26] Document the semantics of parentInputAttrPath It's the input attribute path, relative to the top-level flake, of the flake that *declares* the relative path (i.e. the flake that declares the override, in the case of overridden inputs), which is not necessarily the input's parent in the lock file graph. Assisted-by: Claude Fable 5 --- src/libflake/include/nix/flake/flake.hh | 8 ++++++-- src/libflake/lockfile-v7.cc | 3 --- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 72894f3d7312..ba4514f48067 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -185,8 +185,12 @@ struct LockedFlake bool buildTime = false; /** - * For relative path inputs, the input attribute path of the - * flake relative to which the path is interpreted. + * For relative path inputs (e.g. 'path:./foo'), the input + * attribute path, relative to the top-level flake, of the + * flake against whose source tree the path is resolved. This + * is the flake whose `flake.nix` *declares* the relative + * path: for an overridden input, that's the flake that + * declares the override, not necessarily the input's parent. */ std::optional parentInputAttrPath; }; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index ca64a007648a..90163163036f 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -94,9 +94,6 @@ struct LockedNode : Node FlakeRef lockedRef, originalRef; bool isFlake = true; bool buildTime = false; - - /* The node relative to which relative source paths - (e.g. 'path:../foo') are interpreted. */ std::optional parentInputAttrPath; /** From 5606148eb6f7d5bd84a61e1ce2e66e5b14d72825 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 1 Aug 2026 11:54:51 +0200 Subject: [PATCH 11/26] Make LockFlags::inputUpdates an optional set `std::nullopt` now means "update all inputs". Bare `nix flake update` uses this instead of setting `recreateLockFile`, so recreating the lock file is now only triggered by the explicit `--recreate-lock-file` flag (which is hereby un-deprecated). This distinction matters for the upcoming lock file version dispatch: `nix flake update` will keep the lock file's existing format, while `--recreate-lock-file` will switch it to the configured format. Note that "update all inputs" is implemented by ignoring the *top-level* old lock file (as `recreateLockFile` did), not by refusing reuse at every path: entries seeded from a dependency's own lock file (e.g. a vendored subflake) must still be copied from it. Assisted-by: Claude Fable 5 --- src/libcmd/installables.cc | 16 +++------------- src/libflake/include/nix/flake/flake.hh | 5 +++-- src/libflake/lockfile-v7.cc | 21 +++++++++++++-------- src/nix/develop.cc | 2 +- src/nix/flake.cc | 10 ++++++---- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 3cc343faccf2..1f2b1f1d3534 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -48,19 +48,9 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = R"( - Recreate the flake's lock file from scratch. - - > **DEPRECATED** - > - > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. - )", + .description = "Recreate the flake's lock file from scratch.", .category = category, - .handler = {[&]() { - lockFlags.recreateLockFile = true; - warn( - "'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); - }}, + .handler = {&lockFlags.recreateLockFile, true}, }); addFlag({ @@ -117,7 +107,7 @@ MixFlakeOptions::MixFlakeOptions() if (!path) throw UsageError( "--update-input was passed a zero-length input path, which would refer to the flake itself, not an input"); - lockFlags.inputUpdates.insert(*path); + lockFlags.inputUpdates->insert(*path); }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index ba4514f48067..4f53b9e6c9b8 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -324,9 +324,10 @@ struct LockFlags /** * Flake inputs to be updated. This means that any existing lock - * for those inputs will be ignored. + * for those inputs will be ignored. `std::nullopt` means that + * *all* inputs will be updated. */ - std::set inputUpdates; + std::optional> inputUpdates = std::set(); /** * Whether to require a locked input. diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 90163163036f..902ef8b3609e 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -795,7 +795,7 @@ std::unique_ptr lockFlakeV7( updatesUsed.insert(inputAttrPath); - if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath)) + if (oldNode && !(lockFlags.inputUpdates && lockFlags.inputUpdates->count(nonEmptyInputAttrPath))) if (auto oldLock2 = get(oldNode->inputs, id)) if (auto oldLock3 = std::get_if<0>(&*oldLock2)) oldLock = *oldLock3; @@ -818,10 +818,14 @@ std::unique_ptr lockFlakeV7( /* If we have this input in updateInputs, then we must fetch the flake to update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath); + auto mustRefetch = false; - auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size() - && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); + if (lockFlags.inputUpdates) { + auto lb = lockFlags.inputUpdates->lower_bound(nonEmptyInputAttrPath); + + mustRefetch = lb != lockFlags.inputUpdates->end() && lb->get().size() > inputAttrPath.size() + && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); + } FlakeInputs fakeInputs; @@ -993,7 +997,7 @@ std::unique_ptr lockFlakeV7( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), + lockFlags.recreateLockFile || !lockFlags.inputUpdates ? nullptr : oldLockFile.root.get_ptr(), {}, flake.path, false); @@ -1002,9 +1006,10 @@ std::unique_ptr lockFlakeV7( if (!overridesUsed.count(i.first)) warn("the flag '--override-input %s %s' does not match any input", printInputAttrPath(i.first), i.second); - for (auto & i : lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("'%s' does not match any input of this flake", printInputAttrPath(i)); + if (lockFlags.inputUpdates) + for (auto & i : *lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("'%s' does not match any input of this flake", printInputAttrPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ec190d26f6fe..5295a963bf35 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -641,7 +641,7 @@ struct CmdDevelop : Common, MixEnvironment auto nixpkgsLockFlags = lockFlags; nixpkgsLockFlags.inputOverrides = {}; - nixpkgsLockFlags.inputUpdates = {}; + nixpkgsLockFlags.inputUpdates = std::set(); auto nixpkgs = defaultNixpkgsFlakeRef(); if (auto * i = dynamic_cast(&*installable)) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6da6c650449e..c2c144b973a9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -106,11 +106,11 @@ struct CmdFlakeUpdate : FlakeCommand inputToUpdate); throw e; } - if (lockFlags.inputUpdates.contains(*inputAttrPath)) + if (lockFlags.inputUpdates->contains(*inputAttrPath)) warn( "Input '%s' was specified multiple times. You may have done this by accident.", printInputAttrPath(*inputAttrPath)); - lockFlags.inputUpdates.insert(*inputAttrPath); + lockFlags.inputUpdates->insert(*inputAttrPath); } }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { @@ -133,9 +133,11 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { fetchSettings.tarballTtl = 0; - auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = updateAll; + /* If no specific inputs are given, update all inputs. */ + if (lockFlags.inputUpdates->empty()) + lockFlags.inputUpdates = std::nullopt; + lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; lockFlags.requireLockable = false; From ab8592cd21d35d9f578377edf6f98943711e3259 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 11:42:21 +0200 Subject: [PATCH 12/26] libflake: Add lock-file-format setting Selects the lock file format version (currently 7, the default, or 8) used when creating a new lock file. An existing lock file keeps its version unless `--recreate-lock-file` is passed. Settable via nix.conf, `--option lock-file-format N` or the auto-generated `--lock-file-format` flag. Assisted-by: Claude Fable 5 --- src/libflake/include/nix/flake/settings.hh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 05b36f5b779c..4a9f264d8aee 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -41,6 +41,18 @@ struct Settings : public Config )", {"commit-lockfile-summary"}, true}; + + Setting lockFileFormat{ + this, + 7, + "lock-file-format", + R"( + The lock file format version to use when creating a new lock + file (7 or 8). An existing lock file keeps its version + unless `--recreate-lock-file` is passed. + )", + {}, + true}; }; } // namespace nix::flake From c74de77b3ac90cdf5f6034f9f326b18a3b586cc3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 11:52:18 +0200 Subject: [PATCH 13/26] Move unused override/update warnings from lockFlakeV7() into lockFlake() lockFlakeV7() now returns a LockFlakeResult struct containing the locked flake and the overridesUsed/updatesUsed sets, so the warnings about unmatched '--override-input' / update flags can be emitted by lockFlake(). This allows the upcoming lockFlakeV8() to share that code. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 17 +++++++++++++++-- src/libflake/include/nix/flake/flake.hh | 23 ++++++++++++++++++++++- src/libflake/lockfile-v7.cc | 17 ++++++----------- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 1f84c50dda98..e610a254f602 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -456,7 +456,20 @@ std::unique_ptr lockFlake( debug("old lock file: %s", oldLockedFlake->to_string()); - auto lockedFlake = lockFlakeV7(settings, state, lockFlags, std::move(flake), *oldLockedFlake); + auto [lockedFlake, overridesUsed, updatesUsed] = + lockFlakeV7(settings, state, lockFlags, std::move(flake), *oldLockedFlake); + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn( + "the flag '--override-input %s %s' does not match any input", + printInputAttrPath(i.first), + i.second); + + if (lockFlags.inputUpdates) + for (auto & i : *lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("'%s' does not match any input of this flake", printInputAttrPath(i)); debug("new lock file: %s", lockedFlake->to_string()); @@ -550,7 +563,7 @@ std::unique_ptr lockFlake( } } - return lockedFlake; + return std::move(lockedFlake); } catch (Error & e) { e.addTrace({}, "while updating the lock file of flake '%s'", flakeRefForTrace); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 4f53b9e6c9b8..134e8cc7b7a4 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -347,6 +347,27 @@ Flake readFlake( const SourcePath & rootDir, const InputAttrPath & lockRootPath); +/** + * The result of functions like `lockFlakeV7()` that compute a lock + * file for a flake. + */ +struct LockFlakeResult +{ + std::unique_ptr lockedFlake; + + /** + * The elements of `LockFlags::inputOverrides` that matched an + * input of the flake. + */ + std::set overridesUsed; + + /** + * The elements of `LockFlags::inputUpdates` that matched an input + * of the flake. + */ + std::set updatesUsed; +}; + /* * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is * writable. @@ -372,7 +393,7 @@ std::unique_ptr parseLockFileV7( * `oldLockFile` (which must have been produced by `parseLockFileV7()`) * where possible. Note: this does not write the new lock file. */ -std::unique_ptr lockFlakeV7( +LockFlakeResult lockFlakeV7( const Settings & settings, EvalState & state, const LockFlags & lockFlags, diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 902ef8b3609e..e710d409e61d 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -612,7 +612,7 @@ std::unique_ptr parseLockFileV7( return std::make_unique(fetchSettings, std::move(flake), json, path); } -std::unique_ptr lockFlakeV7( +LockFlakeResult lockFlakeV7( const Settings & settings, EvalState & state, const LockFlags & lockFlags, @@ -1002,19 +1002,14 @@ std::unique_ptr lockFlakeV7( flake.path, false); - for (auto & i : lockFlags.inputOverrides) - if (!overridesUsed.count(i.first)) - warn("the flag '--override-input %s %s' does not match any input", printInputAttrPath(i.first), i.second); - - if (lockFlags.inputUpdates) - for (auto & i : *lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("'%s' does not match any input of this flake", printInputAttrPath(i)); - /* Check 'follows' inputs. */ newLockFile.check(); - return std::make_unique(std::move(flake), std::move(newLockFile)); + return { + .lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)), + .overridesUsed = std::move(overridesUsed), + .updatesUsed = std::move(updatesUsed), + }; } } // namespace nix::flake From e767770534df7dbfa6ffcda6bcc3ff2b99988e1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 14:06:53 +0200 Subject: [PATCH 14/26] libflake: Replace the per-version diff() methods with a generic diffLockedFlakes() The new free function diffLockedFlakes() (in diff.cc) can diff two LockedFlakes of any version. It relies on two new LockedFlake virtual methods: version(), used to report lock file version changes (which previously happened in lockFlake()), and getAllLockEntries(), which flattens the contents of a lock file into a map from input attribute paths to locked flakerefs or 'follows' targets. A nice side effect is that migrating an unchanged lock file between versions 7 and 8 now shows just the version change, rather than every input as "added". getAllLockEntries() takes a `fetchTransitive` flag (currently unimplemented for version 8) that will allow a future `nix flake diff-locks` command to include the transitive locks of inputs that have a lock file of their own. Assisted-by: Claude Fable 5 --- src/libflake/diff.cc | 76 +++++++++++++++++++++ src/libflake/flake.cc | 2 +- src/libflake/include/nix/flake/flake.hh | 31 ++++++++- src/libflake/lockfile-v7.cc | 87 +++++++------------------ src/libflake/meson.build | 1 + 5 files changed, 129 insertions(+), 68 deletions(-) create mode 100644 src/libflake/diff.cc diff --git a/src/libflake/diff.cc b/src/libflake/diff.cc new file mode 100644 index 000000000000..6e585a1ec6e3 --- /dev/null +++ b/src/libflake/diff.cc @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include + +#include "nix/flake/flake.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/flake/flakeref.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/fmt.hh" + +namespace nix::flake { + +static std::string describe(const FlakeRef & flakeRef) +{ + auto s = fmt("'%s'", flakeRef.to_string(true)); + + if (auto lastModified = flakeRef.input.getLastModified()) + s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); + + return s; +} + +static std::string describe(const LockedFlake::LockEntry & entry) +{ + if (auto lockedRef = std::get_if(&entry)) + return describe(*lockedRef); + else + return fmt("follows '%s'", printInputAttrPath(std::get(entry))); +} + +std::string +diffLockedFlakes(const LockedFlake & oldLockedFlake, const LockedFlake & newLockedFlake, bool fetchTransitive) +{ + std::string res; + + if (oldLockedFlake.version() != newLockedFlake.version()) + res += + fmt("• " ANSI_BOLD "Updated lock file version from %d to %d" ANSI_NORMAL "\n", + oldLockedFlake.version(), + newLockedFlake.version()); + + auto oldFlat = oldLockedFlake.getAllLockEntries(fetchTransitive); + auto newFlat = newLockedFlake.getAllLockEntries(fetchTransitive); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += + fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", + printInputAttrPath(j->first), + describe(j->second)); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); + ++i; + } else { + if (i->second != j->second) { + res += + fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", + printInputAttrPath(i->first), + describe(i->second), + describe(j->second)); + } + ++i; + ++j; + } + } + + return res; +} + +} // namespace nix::flake diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index e610a254f602..070887a09b0a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -479,7 +479,7 @@ std::unique_ptr lockFlake( auto lockedFlakeJson = lockedFlake->toJSON(); if (lockedFlakeJson != oldLockedFlake->toJSON() || lockFlags.outputLockFilePath) { - auto diff = lockedFlake->diff(*oldLockedFlake); + auto diff = diffLockedFlakes(*oldLockedFlake, *lockedFlake, false); if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 134e8cc7b7a4..f4558b613330 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -241,10 +241,25 @@ struct LockedFlake virtual std::optional isUnlocked(const fetchers::Settings & fetchSettings) const = 0; /** - * Return a human-readable description of the differences between - * the (older) `oldLockFile` and the lock file of this flake. + * Return the version of this lock file's format (e.g. 7 or 8). */ - virtual std::string diff(const LockedFlake & oldLockFile) const = 0; + virtual unsigned int version() const = 0; + + /** + * A lock file entry: either the locked flakeref of an input, or, + * for "follows" inputs, the input attribute path of the target of + * the "follows" (relative to the top-level flake). + */ + using LockEntry = std::variant; + + /** + * Return the contents of this lock file as a map from input + * attribute paths to lock entries. If `fetchTransitive` is true, + * inputs that have a lock file of their own may be fetched in + * order to include their transitive locks; otherwise only the + * locks contained in this lock file are returned. + */ + virtual std::map getAllLockEntries(bool fetchTransitive) const = 0; virtual nlohmann::json toJSON() const = 0; @@ -253,6 +268,16 @@ struct LockedFlake std::ostream & operator<<(std::ostream & stream, const LockedFlake & lockedFlake); +/** + * Return a human-readable description of the differences between two + * locked flakes (which may use different lock file versions), e.g. + * between the old and new version of a lock file written by + * `lockFlake()`. If `fetchTransitive` is true, transitive lock files + * may be fetched (see `LockedFlake::getAllLockEntries()`). + */ +std::string +diffLockedFlakes(const LockedFlake & oldLockedFlake, const LockedFlake & newLockedFlake, bool fetchTransitive); + struct LockFlags { /** diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index e710d409e61d..7ce6de355f88 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -183,6 +183,12 @@ struct LockFileV7 { ref root = make_ref(); + /** + * The version of the lock file this was parsed from (5-7), or 7 + * for new lock files. + */ + unsigned int version = 7; + LockFileV7() {}; LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) @@ -191,6 +197,8 @@ struct LockFileV7 if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); + this->version = version; + std::string rootKey = json["root"]; std::map> nodeMap{{rootKey, root}}; @@ -383,36 +391,6 @@ struct LockFileV7 } }; -static std::string describe(const FlakeRef & flakeRef) -{ - auto s = fmt("'%s'", flakeRef.to_string(true)); - - if (auto lastModified = flakeRef.input.getLastModified()) - s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); - - return s; -} - -std::ostream & operator<<(std::ostream & stream, const Node::Edge & edge) -{ - if (auto node = std::get_if<0>(&edge)) - stream << describe((*node)->lockedRef); - else if (auto follows = std::get_if<1>(&edge)) - stream << fmt("follows '%s'", printInputAttrPath(*follows)); - return stream; -} - -static bool equals(const Node::Edge & e1, const Node::Edge & e2) -{ - if (auto n1 = std::get_if<0>(&e1)) - if (auto n2 = std::get_if<0>(&e2)) - return (*n1)->lockedRef == (*n2)->lockedRef; - if (auto f1 = std::get_if<1>(&e1)) - if (auto f2 = std::get_if<1>(&e2)) - return *f1 == *f2; - return false; -} - struct LockedFlakeV7 : LockedFlake { /** @@ -544,41 +522,22 @@ struct LockedFlakeV7 : LockedFlake return lockFile.isUnlocked(fetchSettings); } - std::string diff(const LockedFlake & _oldLockFile) const override + unsigned int version() const override { - /* If `oldLockFile` is not a version 7 lock file, diff against an - empty lock file, i.e. all inputs of this lock file will show up - as added. */ - auto oldLockFile = dynamic_cast(&_oldLockFile); - - auto oldFlat = oldLockFile ? oldLockFile->lockFile.getAllInputs() : std::map(); - auto newFlat = lockFile.getAllInputs(); - - auto i = oldFlat.begin(); - auto j = newFlat.begin(); - std::string res; - - while (i != oldFlat.end() || j != newFlat.end()) { - if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { - res += - fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", - printInputAttrPath(j->first), - j->second); - ++j; - } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { - res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); - ++i; - } else { - if (!equals(i->second, j->second)) { - res += - fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", - printInputAttrPath(i->first), - i->second, - j->second); - } - ++i; - ++j; - } + return lockFile.version; + } + + std::map getAllLockEntries(bool fetchTransitive) const override + { + /* Note: `fetchTransitive` is irrelevant, since a version 7 + lock file already contains all transitive locks. */ + std::map res; + + for (auto & [path, edge] : lockFile.getAllInputs()) { + if (auto node = std::get_if<0>(&edge)) + res.emplace(path, (*node)->lockedRef); + else + res.emplace(path, std::get<1>(edge)); } return res; diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 33de5958c8c8..a7dee62b306c 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -41,6 +41,7 @@ endforeach sources = files( 'config.cc', + 'diff.cc', 'flake-primops.cc', 'flake.cc', 'flakeref.cc', From cb3daf59e3ca6ad802847dcbb9e36372e3bf077f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 15:41:54 +0200 Subject: [PATCH 15/26] Add 'nix flake diff-locks' command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shows the differences between the lock files of two flakes, e.g. # nix flake diff-locks github:NixOS/nix/2.28.0 github:NixOS/nix/2.29.0 • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/48d12d5' (2024-12-16) → 'github:NixOS/nixpkgs/adaa24f' (2025-05-13) The second argument defaults to the flake in the current directory. The flakes do not need to use the same lock file format version. The --transitive flag (not yet supported for version 8 lock files) includes the transitive locks of inputs that have a lock file of their own. Also factors the version dispatch in lockFlake() into a new generic parseLockFile() function that the command uses to read lock files of any version. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 17 ++++- src/libflake/include/nix/flake/flake.hh | 13 ++++ src/nix/flake-diff-locks.cc | 95 +++++++++++++++++++++++++ src/nix/flake-diff-locks.md | 34 +++++++++ src/nix/meson.build | 2 +- 5 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 src/nix/flake-diff-locks.cc create mode 100644 src/nix/flake-diff-locks.md diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 070887a09b0a..f2d9f1e2e9a7 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -421,6 +421,21 @@ Flake getFlake( return getFlake(state, originalRef, useRegistries, {}, requireLockable); } +std::unique_ptr parseLockFile( + const fetchers::Settings & fetchSettings, + Flake flake, + const nlohmann::json & json, + std::string_view path, + unsigned int versionIfMissing) +{ + auto version = json.is_null() ? versionIfMissing : (unsigned int) json.value("version", 0); + + if (version >= 5 && version <= 7) + return parseLockFileV7(fetchSettings, std::move(flake), json, path); + else + throw Error("lock file '%s' has unsupported version %d", path, version); +} + std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { @@ -452,7 +467,7 @@ std::unique_ptr lockFlake( } // FIXME: dispatch on the lock file version here. - auto oldLockedFlake = parseLockFileV7(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); + auto oldLockedFlake = parseLockFile(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); debug("old lock file: %s", oldLockedFlake->to_string()); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index f4558b613330..a0d372712643 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -406,6 +406,19 @@ std::unique_ptr lockFlake( std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); +/** + * Parse a lock file, dispatching on the version of its JSON + * representation. `json` must be null if the lock file doesn't exist, + * in which case an empty lock file of version `versionIfMissing` is + * returned. + */ +std::unique_ptr parseLockFile( + const fetchers::Settings & fetchSettings, + Flake flake, + const nlohmann::json & json, + std::string_view path, + unsigned int versionIfMissing = 7); + /** * Parse a lock file in the old graph-based format (versions 5-7). * `json` must be null if the lock file doesn't exist. diff --git a/src/nix/flake-diff-locks.cc b/src/nix/flake-diff-locks.cc new file mode 100644 index 000000000000..448e203998f0 --- /dev/null +++ b/src/nix/flake-diff-locks.cc @@ -0,0 +1,95 @@ +#include "flake-command.hh" +#include "nix/expr/eval.hh" +#include "nix/fetchers/fetch-settings.hh" + +#include +#include + +namespace nix { + +using namespace flake; + +struct CmdFlakeDiffLocks : EvalCommand +{ + std::string oldFlakeUrl, newFlakeUrl = "."; + + bool transitive = false; + + CmdFlakeDiffLocks() + { + expectArgs( + {.label = "old-flake", + .handler = {&oldFlakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); + + expectArgs( + {.label = "new-flake", + .optional = true, + .handler = {&newFlakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); + + addFlag({ + .longName = "transitive", + .description = "Include the transitive locks of inputs that have a lock file of their own. " + "This may require fetching those inputs.", + .handler = {&transitive, true}, + }); + } + + std::string description() override + { + return "show the differences between the lock files of two flakes"; + } + + std::string doc() override + { + return +#include "flake-diff-locks.md" + ; + } + + void run(nix::ref store) override + { + auto state = getEvalState(); + + auto readLockedFlake = [&](const std::string & flakeUrl) { + auto flake = getFlake( + *state, + parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()), + fetchers::UseRegistries::All, + false); + + nlohmann::json json; + + auto lockFilePath = flake.lockFilePath(); + if (lockFilePath.pathExists()) { + try { + json = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + } else + warn("flake '%s' does not have a lock file", flake.originalRef); + + return parseLockFile(state->fetchSettings, std::move(flake), json, fmt("%s", lockFilePath)); + }; + + auto oldLockedFlake = readLockedFlake(oldFlakeUrl); + auto newLockedFlake = readLockedFlake(newFlakeUrl); + + auto diff = diffLockedFlakes(*oldLockedFlake, *newLockedFlake, transitive); + + if (diff.empty()) + logger->cout("No changes."); + else + logger->cout("%s", chomp(diff)); + } +}; + +static auto rCmdFlakeDiffLocks = registerCommand2({"flake", "diff-locks"}); + +} // namespace nix diff --git a/src/nix/flake-diff-locks.md b/src/nix/flake-diff-locks.md new file mode 100644 index 000000000000..cad7f46bbfd4 --- /dev/null +++ b/src/nix/flake-diff-locks.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Show how the locked inputs of two versions of a flake differ: + + ```console + # nix flake diff-locks github:NixOS/nix/2.28.0 github:NixOS/nix/2.29.0 + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/48d12d5' (2024-12-16) + → 'github:NixOS/nixpkgs/adaa24f' (2025-05-13) + ``` + +* Show what has changed in the lock file of the flake in the worktree relative to a previous Git revision: + + ```console + # nix flake diff-locks '.?rev=26842787496f2293c676fb36db38dacfd63497e0' + ``` + + or relative to a Git ref: + + ```console + # nix flake diff-locks '.?ref=HEAD' + ``` + +# Description + +This command shows the differences between the lock files of two flakes *old-flake* and *new-flake*: inputs that were added, removed or updated. *new-flake* defaults to the flake in the current directory. +The flakes do not need to use the same lock file format version; a change in the lock file version is also reported. + +By default, only the locks contained in the two flakes' own lock files are compared. +With `--transitive`, inputs that have a lock file of their own are fetched in order to include their transitive locks in the comparison. + +)"" diff --git a/src/nix/meson.build b/src/nix/meson.build index 9515101546c0..c854206dc19a 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -109,7 +109,7 @@ nix_sources = [ config_priv_h ] + files( 'edit.cc', 'env.cc', 'eval.cc', - 'flake-prefetch-inputs.cc', + 'flake-diff-locks.cc', 'flake-prefetch-inputs.cc', 'flake.cc', 'formatter.cc', From 6ae378cc9812a0e9cae546113fcc7a646f5f913f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 13:01:41 +0200 Subject: [PATCH 16/26] libflake: Factor out the warnRegistry() function This function will be identitical between lockFlakeV7() and lockFlakeV8(), so move it into a utility function in the new internal header flake-impl.hh (it's not part of the public libflake API). Assisted-by: Claude Fable 5 --- src/libflake/flake-impl.hh | 25 +++++++++++++++++++++++++ src/libflake/flake.cc | 26 ++++++++++++++++++++++++++ src/libflake/lockfile-v7.cc | 27 +++------------------------ 3 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 src/libflake/flake-impl.hh diff --git a/src/libflake/flake-impl.hh b/src/libflake/flake-impl.hh new file mode 100644 index 000000000000..52a1469a8a65 --- /dev/null +++ b/src/libflake/flake-impl.hh @@ -0,0 +1,25 @@ +#pragma once +///@file +/// Internal declarations shared between the lock file +/// implementations. Not part of the public libflake API. + +#include "nix/flake/flakeref.hh" +#include "nix/flake/input-attr-path.hh" +#include "nix/util/source-path.hh" + +namespace nix::flake { + +/** + * Warn against the use of indirect flakerefs (but only for top-level + * inputs, since we don't want to annoy users about flakes that are + * not under their control). `ref` is the input's flakeref as declared + * and `resolvedRef` its registry-resolved counterpart; `topFlakePath` + * is the `flake.nix` of the top-level flake. + */ +void warnRegistry( + const InputAttrPath & inputAttrPath, + const FlakeRef & ref, + const FlakeRef & resolvedRef, + const SourcePath & topFlakePath); + +} // namespace nix::flake diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index f2d9f1e2e9a7..b7b06eaa688e 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "nix/util/ref.hh" #include "nix/util/environment-variables.hh" #include "nix/flake/flake.hh" +#include "flake-impl.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-cache.hh" #include "nix/expr/eval-settings.hh" @@ -41,6 +43,7 @@ #include "nix/expr/fetch-tree.hh" #include "nix/expr/json-to-value.hh" #include "nix/expr/primops.hh" +#include "nix/expr/print.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -436,6 +439,29 @@ std::unique_ptr parseLockFile( throw Error("lock file '%s' has unsupported version %d", path, version); } +void warnRegistry( + const InputAttrPath & inputAttrPath, + const FlakeRef & ref, + const FlakeRef & resolvedRef, + const SourcePath & topFlakePath) +{ + if (inputAttrPath.size() == 1 && !ref.input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + printInputAttrPath(inputAttrPath), + topFlakePath, + s.str()); + } +} + std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 7ce6de355f88..e56990969c22 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -22,6 +22,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/flake.hh" +#include "flake-impl.hh" #include "nix/util/sync.hh" #include "nix/flake/settings.hh" #include "nix/expr/eval.hh" @@ -862,28 +863,6 @@ LockFlakeResult lockFlakeV7( auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; - /* Warn against the use of indirect flakerefs - (but only at top-level since we don't want - to annoy users about flakes that are not - under their control). */ - auto warnRegistry = [&](const FlakeRef & resolvedRef) { - if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { - std::ostringstream s; - printLiteralString(s, resolvedRef.to_string()); - warn( - "Flake input '%1%' uses the flake registry. " - "Using the registry in flake inputs is deprecated in Determinate Nix. " - "To make your flake future-proof, add the following to '%2%':\n" - "\n" - " inputs.%1%.url = %3%;\n" - "\n" - "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", - inputAttrPathS, - flake.path, - s.str()); - } - }; - if (input.isFlake) { auto inputFlake = getInputFlake( *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); @@ -912,7 +891,7 @@ LockFlakeResult lockFlakeV7( inputFlake.path, false); - warnRegistry(inputFlake.resolvedRef); + warnRegistry(inputAttrPath, *input.ref, inputFlake.resolvedRef, flake.path); } else { @@ -927,7 +906,7 @@ LockFlakeResult lockFlakeV7( auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - warnRegistry(resolvedRef); + warnRegistry(inputAttrPath, *input.ref, resolvedRef, flake.path); return { state.storePath(state.mountInput( From 51da5d1687c9f4db4bab2626d7641fd0aa043930 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 17:39:42 +0200 Subject: [PATCH 17/26] Make LockedFlake::visit() non-virtual It can be implemented generically on top of getInputTargets() (which provides the input names and 'follows' targets of each level) and findInput() (which provides the InputInfo of non-follows inputs), so the per-version implementations are unnecessary. Note that we now only recurse into inputs with `isFlake = true`; in a well-formed version 7 lock file, non-flake nodes never have children anyway. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 19 +++++++++++++++++++ src/libflake/include/nix/flake/flake.hh | 2 +- src/libflake/lockfile-v7.cc | 25 ------------------------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index b7b06eaa688e..7ed1e2d829bd 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -837,6 +837,25 @@ std::vector LockedFlake::getInputNames(const InputAttrPath & prefix) co return res; } +void LockedFlake::visit(VisitCallback callback) const +{ + if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) + return; + + [&](this const auto & recurse, const InputAttrPath & prefix) -> void { + for (auto & [id, target] : getInputTargets(prefix)) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + if (target) + callback(inputAttrPath, *target); + else if (auto info = findInput(inputAttrPath)) { + if (callback(inputAttrPath, *info) && info->isFlake) + recurse(inputAttrPath); + } + } + }({}); +} + std::string LockedFlake::to_string() const { return toJSON().dump(2); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index a0d372712643..2f8736b0ee1c 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -230,7 +230,7 @@ struct LockedFlake * the inputs of that input. We never recurse into "follows" * inputs; their targets are visited under their own paths. */ - virtual void visit(VisitCallback callback) const = 0; + void visit(VisitCallback callback) const; std::optional getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index e56990969c22..41c08fa3dfc4 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -493,31 +493,6 @@ struct LockedFlakeV7 : LockedFlake return path; } - void visit(VisitCallback callback) const override - { - if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) - return; - - [&](this const auto & recurse, const InputAttrPath & prefix, ref node) -> void { - for (auto & [id, input] : node->inputs) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(id); - if (auto child = std::get_if<0>(&input)) { - if (callback( - inputAttrPath, - InputInfo{ - .lockedRef = (*child)->lockedRef, - .isFlake = (*child)->isFlake, - .buildTime = (*child)->buildTime, - })) - recurse(inputAttrPath, *child); - } else if (auto follows = std::get_if<1>(&input)) { - callback(inputAttrPath, *follows); - } - } - }({}, lockFile.root); - } - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override { return lockFile.isUnlocked(fetchSettings); From 20f20a6743c9314d32b410dfb58c5de606a226cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 18:38:51 +0200 Subject: [PATCH 18/26] LockedFlake: Require fully resolved paths in the query methods getInputTargets(), findInput() and getSourcePath() now require an input attribute path that doesn't pass through any 'follows' input, and throw an error otherwise. The new non-virtual resolveFollows() method resolves a path into that form, implemented generically on top of getInputTargets() (which returns the immediate 'follows' targets). This simplifies the per-version implementations: LockFileV7's node lookup no longer resolves 'follows' edges (and the check that 'follows' targets exist, previously the other user of that resolution, now uses resolveFollows() in lockFlakeV7()). The only external callers that need resolveFollows() are '--inputs-from' and InstallableFlake::nixpkgsFlakeRef(), where the named input may be a 'follows'; call-flake.nix and visit() only ever recurse into resolved paths. Assisted-by: Claude Fable 5 --- src/libcmd/installable-flake.cc | 4 +- src/libcmd/installables.cc | 3 +- src/libflake/flake.cc | 47 +++++++++++++ src/libflake/include/nix/flake/flake.hh | 32 ++++++--- src/libflake/lockfile-v7.cc | 89 ++++++++++--------------- 5 files changed, 108 insertions(+), 67 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 778c013980cb..46f39172e61f 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -327,7 +327,9 @@ ref InstallableFlake::openEvalCache() const FlakeRef InstallableFlake::nixpkgsFlakeRef() const { - if (auto nixpkgsInput = getLockedFlake()->findInput({"nixpkgs"})) { + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->findInput(lockedFlake->resolveFollows({"nixpkgs"}))) { if (nixpkgsInput->isFlake) { debug("using nixpkgs flake '%s'", nixpkgsInput->lockedRef); return std::move(nixpkgsInput->lockedRef); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1f2b1f1d3534..4a8515001755 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -172,8 +172,7 @@ MixFlakeOptions::MixFlakeOptions() {.writeLockFile = false}); for (auto & inputName : lockedFlake->getInputNames({})) { - // Note: findInput() resolves 'follows' nodes. - if (auto input = lockedFlake->findInput({inputName})) { + if (auto input = lockedFlake->findInput(lockedFlake->resolveFollows({inputName}))) { fetchers::Attrs extraAttrs; if (!input->lockedRef.subdir.empty()) { diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 7ed1e2d829bd..bf73bd76d784 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -63,6 +63,7 @@ #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" #include "nix/util/source-path.hh" +#include "nix/util/strings.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" @@ -837,6 +838,52 @@ std::vector LockedFlake::getInputNames(const InputAttrPath & prefix) co return res; } +InputAttrPath LockedFlake::resolveFollows(const InputAttrPath & path) const +{ + std::vector visited; + + InputAttrPath res; + + /* The path elements still to be resolved, in reverse order. */ + InputAttrPath todo(path.rbegin(), path.rend()); + + while (!todo.empty()) { + auto name = todo.back(); + todo.pop_back(); + + auto targets = getInputTargets(res); + auto i = targets.find(name); + + if (i == targets.end()) { + /* The input doesn't exist, so there is nothing to + resolve; return the remaining path unchanged and leave + it to the caller to deal with it. */ + res.push_back(std::move(name)); + res.insert(res.end(), todo.rbegin(), todo.rend()); + return res; + } + + if (i->second) { + /* A "follows" input: restart resolution from its target + (which is relative to the top-level flake). */ + auto followsPath(res); + followsPath.push_back(std::move(name)); + if (std::find(visited.begin(), visited.end(), followsPath) != visited.end()) { + std::vector cycle; + std::transform(visited.begin(), visited.end(), std::back_inserter(cycle), printInputAttrPath); + cycle.push_back(printInputAttrPath(followsPath)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(std::move(followsPath)); + todo.insert(todo.end(), i->second->rbegin(), i->second->rend()); + res.clear(); + } else + res.push_back(std::move(name)); + } + + return res; +} + void LockedFlake::visit(VisitCallback callback) const { if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 2f8736b0ee1c..b9111e60b7f5 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -164,17 +164,29 @@ struct LockedFlake * for a "follows" input, the input attribute path (relative to * the top-level flake) of the immediate target of the * "follows". Note that the target may itself denote a "follows" - * input. Throws an error if `prefix` does not denote an existing - * input. + * input. `prefix` must be fully resolved (see + * `resolveFollows()`). Throws an error if `prefix` does not + * denote an existing input. */ virtual std::map> getInputTargets(const InputAttrPath & prefix) const = 0; /** * Return the names of the inputs of the input denoted by * `prefix`, or of the top-level flake if `prefix` is empty. + * `prefix` must be fully resolved (see `resolveFollows()`). */ std::vector getInputNames(const InputAttrPath & prefix) const; + /** + * Resolve any "follows" indirections in `path`, returning an + * input attribute path that denotes the same input but does not + * pass through any "follows" input. Such a *fully resolved* path + * is required by methods like `getInputTargets()`, `findInput()` + * and `getSourcePath()`. Path elements that do not denote + * existing inputs are returned unchanged. + */ + InputAttrPath resolveFollows(const InputAttrPath & path) const; + /** * Information about a locked input. */ @@ -196,19 +208,21 @@ struct LockedFlake }; /** - * Return information about the input denoted by `path`, resolving - * 'follows' indirections. Returns std::nullopt if the input does - * not exist. + * Return information about the input denoted by `path`, which + * must be fully resolved (see `resolveFollows()`); an error is + * thrown if it passes through a "follows" input. Returns + * std::nullopt if the input does not exist. */ virtual std::optional findInput(const InputAttrPath & path) const = 0; /** * Return the source path of the input denoted by `inputAttrPath` * (or of the top-level flake if `inputAttrPath` is empty), - * fetching it if necessary. Note: the returned path is backed by - * `EvalState::rootFS` (i.e. it's a store path, possibly a virtual - * one that has the input's accessor mounted on it if lazy trees - * are enabled), not by the input's original accessor. + * fetching it if necessary. `inputAttrPath` must be fully + * resolved (see `resolveFollows()`). Note: the returned path is + * backed by `EvalState::rootFS` (i.e. it's a store path, possibly + * a virtual one that has the input's accessor mounted on it if + * lazy trees are enabled), not by the input's original accessor. */ virtual SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const = 0; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 41c08fa3dfc4..5d030cc69984 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -145,38 +145,6 @@ struct LockedNode : Node } }; -static std::shared_ptr -doFind(const ref & root, const InputAttrPath & path, std::vector & visited) -{ - auto pos = root; - - auto found = std::find(visited.cbegin(), visited.cend(), path); - - if (found != visited.end()) { - std::vector cycle; - std::transform(found, visited.cend(), std::back_inserter(cycle), printInputAttrPath); - cycle.push_back(printInputAttrPath(path)); - throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); - } - visited.push_back(path); - - for (auto & elem : path) { - if (auto i = get(pos->inputs, elem)) { - if (auto node = std::get_if<0>(&*i)) - pos = *node; - else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = doFind(root, *follows, visited)) - pos = ref(p); - else - return {}; - } - } else - return {}; - } - - return pos; -} - /** * The old graph-based lock file format (versions 5-7). */ @@ -346,10 +314,29 @@ struct LockFileV7 return toJSON() == other.toJSON(); } + /** + * Return the node denoted by `path`, which must be fully + * resolved: an error is thrown if it passes through a 'follows' + * edge. Returns null if the input doesn't exist. + */ std::shared_ptr findInput(const InputAttrPath & path) const { - std::vector visited; - return doFind(root, path, visited); + std::shared_ptr pos = root.get_ptr(); + + for (const auto & [n, elem] : enumerate(path)) { + auto i = get(pos->inputs, elem); + if (!i) + return nullptr; + if (auto node = std::get_if<0>(&*i)) + pos = node->get_ptr(); + else + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath({path.begin(), path.begin() + n + 1})); + } + + return pos; } std::map getAllInputs() const @@ -372,24 +359,6 @@ struct LockFileV7 return res; } - - /** - * Check that every 'follows' input target exists. - */ - void check() - { - auto inputs = getAllInputs(); - - for (auto & [inputAttrPath, input] : inputs) { - if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !findInput(*follows)) - throw Error( - "input '%s' follows a non-existent input '%s'", - printInputAttrPath(inputAttrPath), - printInputAttrPath(*follows)); - } - } - } }; struct LockedFlakeV7 : LockedFlake @@ -915,11 +884,21 @@ LockFlakeResult lockFlakeV7( flake.path, false); - /* Check 'follows' inputs. */ - newLockFile.check(); + auto lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)); + + /* Check that the target of every 'follows' input exists. */ + for (auto & [inputAttrPath, input] : lockedFlake->lockFile.getAllInputs()) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !lockedFlake->lockFile.findInput(lockedFlake->resolveFollows(*follows))) + throw Error( + "input '%s' follows a non-existent input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(*follows)); + } + } return { - .lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)), + .lockedFlake = std::move(lockedFlake), .overridesUsed = std::move(overridesUsed), .updatesUsed = std::move(updatesUsed), }; From 1678bd47ecb9de9ecad8a764930ee67745a9f115 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 16:59:57 +0200 Subject: [PATCH 19/26] Add an EvalState parameter to the LockedFlake query methods getInputTargets(), findInput(), visit() and the non-virtual getInputNames() now take an EvalState, like getSourcePath() already did. The version 7 implementations don't need it (their lock files contain the full dependency graph), but the version 8 implementations will have to fetch inputs and read their flake.nix / flake.lock files on demand. Assisted-by: Claude Fable 5 --- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installables.cc | 5 +++-- src/libflake/flake.cc | 20 ++++++++++---------- src/libflake/include/nix/flake/flake.hh | 11 ++++++----- src/libflake/lockfile-v7.cc | 7 ++++--- src/nix/flake-prefetch-inputs.cc | 6 +++--- src/nix/flake.cc | 4 ++-- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 46f39172e61f..ed646345c1d7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -329,7 +329,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const { auto lockedFlake = getLockedFlake(); - if (auto nixpkgsInput = lockedFlake->findInput(lockedFlake->resolveFollows({"nixpkgs"}))) { + if (auto nixpkgsInput = lockedFlake->findInput(*state, lockedFlake->resolveFollows(*state, {"nixpkgs"}))) { if (nixpkgsInput->isFlake) { debug("using nixpkgs flake '%s'", nixpkgsInput->lockedRef); return std::move(nixpkgsInput->lockedRef); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4a8515001755..bc01a7d5b0e5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -171,8 +171,9 @@ MixFlakeOptions::MixFlakeOptions() parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()), {.writeLockFile = false}); - for (auto & inputName : lockedFlake->getInputNames({})) { - if (auto input = lockedFlake->findInput(lockedFlake->resolveFollows({inputName}))) { + for (auto & inputName : lockedFlake->getInputNames(*evalState, {})) { + if (auto input = + lockedFlake->findInput(*evalState, lockedFlake->resolveFollows(*evalState, {inputName}))) { fetchers::Attrs extraAttrs; if (!input->lockedRef.subdir.empty()) { diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index bf73bd76d784..c4f0828b413e 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -707,7 +707,7 @@ static void prim_listFlakeInputs(EvalState & state, const PosIdx pos, Value ** a auto & lockedFlake = requireLockedFlake(state, *args[0], pos); auto prefix = getInputAttrPathArg(state, *args[1], pos); - auto targets = lockedFlake.getInputTargets(prefix); + auto targets = lockedFlake.getInputTargets(state, prefix); auto attrs = state.buildBindings(targets.size()); @@ -747,7 +747,7 @@ static void prim_fetchFlakeInput(EvalState & state, const PosIdx pos, Value ** a std::optional info; if (!path.empty()) { - info = lockedFlake.findInput(path); + info = lockedFlake.findInput(state, path); if (!info) state.error("flake input '%s' does not exist", printInputAttrPath(path)).atPos(pos).debugThrow(); } @@ -779,7 +779,7 @@ static void prim_fetchFlakeInput(EvalState & state, const PosIdx pos, Value ** a /* The parent is the top-level flake. */ info2.reset(); else - info2 = lockedFlake.findInput(*info2->parentInputAttrPath); + info2 = lockedFlake.findInput(state, *info2->parentInputAttrPath); } emitTreeAttrs( @@ -830,15 +830,15 @@ void callFlake(EvalState & state, std::shared_ptr lockedFlake LockedFlake::~LockedFlake() {} -std::vector LockedFlake::getInputNames(const InputAttrPath & prefix) const +std::vector LockedFlake::getInputNames(EvalState & state, const InputAttrPath & prefix) const { std::vector res; - for (auto & [name, target] : getInputTargets(prefix)) + for (auto & [name, target] : getInputTargets(state, prefix)) res.push_back(name); return res; } -InputAttrPath LockedFlake::resolveFollows(const InputAttrPath & path) const +InputAttrPath LockedFlake::resolveFollows(EvalState & state, const InputAttrPath & path) const { std::vector visited; @@ -851,7 +851,7 @@ InputAttrPath LockedFlake::resolveFollows(const InputAttrPath & path) const auto name = todo.back(); todo.pop_back(); - auto targets = getInputTargets(res); + auto targets = getInputTargets(state, res); auto i = targets.find(name); if (i == targets.end()) { @@ -884,18 +884,18 @@ InputAttrPath LockedFlake::resolveFollows(const InputAttrPath & path) const return res; } -void LockedFlake::visit(VisitCallback callback) const +void LockedFlake::visit(EvalState & state, VisitCallback callback) const { if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) return; [&](this const auto & recurse, const InputAttrPath & prefix) -> void { - for (auto & [id, target] : getInputTargets(prefix)) { + for (auto & [id, target] : getInputTargets(state, prefix)) { auto inputAttrPath(prefix); inputAttrPath.push_back(id); if (target) callback(inputAttrPath, *target); - else if (auto info = findInput(inputAttrPath)) { + else if (auto info = findInput(state, inputAttrPath)) { if (callback(inputAttrPath, *info) && info->isFlake) recurse(inputAttrPath); } diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index b9111e60b7f5..15b0ec9955bc 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -168,14 +168,15 @@ struct LockedFlake * `resolveFollows()`). Throws an error if `prefix` does not * denote an existing input. */ - virtual std::map> getInputTargets(const InputAttrPath & prefix) const = 0; + virtual std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const = 0; /** * Return the names of the inputs of the input denoted by * `prefix`, or of the top-level flake if `prefix` is empty. * `prefix` must be fully resolved (see `resolveFollows()`). */ - std::vector getInputNames(const InputAttrPath & prefix) const; + std::vector getInputNames(EvalState & state, const InputAttrPath & prefix) const; /** * Resolve any "follows" indirections in `path`, returning an @@ -185,7 +186,7 @@ struct LockedFlake * and `getSourcePath()`. Path elements that do not denote * existing inputs are returned unchanged. */ - InputAttrPath resolveFollows(const InputAttrPath & path) const; + InputAttrPath resolveFollows(EvalState & state, const InputAttrPath & path) const; /** * Information about a locked input. @@ -213,7 +214,7 @@ struct LockedFlake * thrown if it passes through a "follows" input. Returns * std::nullopt if the input does not exist. */ - virtual std::optional findInput(const InputAttrPath & path) const = 0; + virtual std::optional findInput(EvalState & state, const InputAttrPath & path) const = 0; /** * Return the source path of the input denoted by `inputAttrPath` @@ -244,7 +245,7 @@ struct LockedFlake * the inputs of that input. We never recurse into "follows" * inputs; their targets are visited under their own paths. */ - void visit(VisitCallback callback) const; + void visit(EvalState & state, VisitCallback callback) const; std::optional getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index 5d030cc69984..f19a48740b9c 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -385,7 +385,8 @@ struct LockedFlakeV7 : LockedFlake { } - std::map> getInputTargets(const InputAttrPath & prefix) const override + std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const override { auto node = lockFile.findInput(prefix); if (!node) @@ -403,7 +404,7 @@ struct LockedFlakeV7 : LockedFlake return res; } - std::optional findInput(const InputAttrPath & path) const override + std::optional findInput(EvalState & state, const InputAttrPath & path) const override { if (auto node = std::dynamic_pointer_cast(lockFile.findInput(path))) return InputInfo{ @@ -889,7 +890,7 @@ LockFlakeResult lockFlakeV7( /* Check that the target of every 'follows' input exists. */ for (auto & [inputAttrPath, input] : lockedFlake->lockFile.getAllInputs()) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !lockedFlake->lockFile.findInput(lockedFlake->resolveFollows(*follows))) + if (!follows->empty() && !lockedFlake->lockFile.findInput(lockedFlake->resolveFollows(state, *follows))) throw Error( "input '%s' follows a non-existent input '%s'", printInputAttrPath(inputAttrPath), diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index e85518a4eb69..33c35ed3ad77 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -25,11 +25,13 @@ struct CmdFlakePrefetchInputs : FlakeCommand { auto flake = lockFlake(); + auto state = getEvalState(); + /* Gather the attribute paths of all transitive inputs, skipping build-time inputs and their dependencies. */ std::vector> inputs; - flake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + flake->visit(*state, [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { auto inputInfo = std::get_if(&input); /* Skip "follows" inputs and build-time inputs (and their @@ -44,8 +46,6 @@ struct CmdFlakePrefetchInputs : FlakeCommand return true; }); - auto state = getEvalState(); - /* Fetch the inputs in parallel. */ ThreadPool pool{fileTransferSettings.httpConnections}; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index c2c144b973a9..0186cb637750 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -265,7 +265,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON TreeNode root; - lockedFlake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + lockedFlake->visit(*getEvalState(), [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { if (!inputAttrPath.empty()) { auto * node = &root; for (auto & elem : inputAttrPath) @@ -840,7 +840,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs return *jsonObj; }; - flake->visit([&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + flake->visit(*getEvalState(), [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { /* Skip "follows" inputs; their targets are visited under their own paths. */ auto inputInfo = std::get_if(&input); From 98b0c4343fbb2095d2ac9281585928f4f4022c4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 20:48:19 +0200 Subject: [PATCH 20/26] LockedFlake::visit(): Work around a GCC internal compiler error GCC segfaults while parsing the recursive lambda with an explicit object parameter ("deducing this") when it contains a call to a member function of the enclosing class. Use the std::function recursion pattern instead. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index c4f0828b413e..dfc2a48f9c66 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -889,7 +889,12 @@ void LockedFlake::visit(EvalState & state, VisitCallback callback) const if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) return; - [&](this const auto & recurse, const InputAttrPath & prefix) -> void { + /* Note: this is not a recursive lambda using an explicit object + parameter because that triggers an internal compiler error in + GCC. */ + std::function recurse; + + recurse = [&](const InputAttrPath & prefix) { for (auto & [id, target] : getInputTargets(state, prefix)) { auto inputAttrPath(prefix); inputAttrPath.push_back(id); @@ -900,7 +905,9 @@ void LockedFlake::visit(EvalState & state, VisitCallback callback) const recurse(inputAttrPath); } } - }({}); + }; + + recurse({}); } std::string LockedFlake::to_string() const From fbc2f7b473b71fc4ed870c31e74226b8b0adc20d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 12:09:29 +0200 Subject: [PATCH 21/26] libflake: Add the version 8 (sparse) lock file data model Add lockfile-v8.cc containing the in-memory representation of the new sparse lock file format: a flat map from slash-separated input attribute paths to locks (with recursive inline `locks` for inputs that lack a lock file of their own), along with JSON parsing, serialization, isUnlocked() and diff(). The eval-time methods (getInputTargets() etc.) and the lock algorithm are stubs for now. Assisted-by: Claude Fable 5 --- src/libflake/include/nix/flake/flake.hh | 20 ++ src/libflake/lockfile-v8.cc | 319 ++++++++++++++++++++++++ src/libflake/meson.build | 1 + 3 files changed, 340 insertions(+) create mode 100644 src/libflake/lockfile-v8.cc diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 15b0ec9955bc..93571a5a3112 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -453,6 +453,26 @@ LockFlakeResult lockFlakeV7( Flake flake, const LockedFlake & oldLockFile); +/** + * Parse a lock file in the sparse format (version 8). `json` must be + * null if the lock file doesn't exist. + */ +std::unique_ptr parseLockFileV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); + +/** + * Compute a version 8 lock file for `flake`, reusing entries from + * `oldLockFile` where possible. If `oldLockFile` was not produced by + * `parseLockFileV8()`, it is ignored. Note: this does not write the + * new lock file. + */ +LockFlakeResult lockFlakeV8( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); + void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & v); } // namespace flake diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc new file mode 100644 index 000000000000..ba5665d5b56c --- /dev/null +++ b/src/libflake/lockfile-v8.cc @@ -0,0 +1,319 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nix/fetchers/fetch-settings.hh" +#include "nix/flake/flake.hh" +#include "flake-impl.hh" +#include "nix/flake/settings.hh" +#include "nix/util/sync.hh" +#include "nix/expr/eval.hh" +#include "nix/store/store-api.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/input-cache.hh" +#include "nix/flake/flakeref.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/error.hh" +#include "nix/util/finally.hh" +#include "nix/util/fmt.hh" +#include "nix/util/logging.hh" +#include "nix/util/types.hh" +#include "nix/util/util.hh" + +namespace nix::flake { + +static FlakeRef getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr) +{ + auto i = json.find(attr); + if (i == json.end()) + throw Error("attribute '%s' missing in lock file", attr); + return FlakeRef::fromAttrs(fetchSettings, fetchers::jsonToAttrs(*i)); +} + +/** + * The sparse lock file format (version 8). Unlike the graph-based + * versions 5-7, it only stores the immediate inputs of the flake, + * plus any overrides of transitive inputs; the rest of the dependency + * graph is resolved at evaluation time from the inputs' own lock + * files. + */ +struct LockFileV8 +{ + struct Lock + { + FlakeRef originalRef, lockedRef; + + /** + * The locks for the transitive inputs of this input. Only + * present if the input does not have a lock file of its + * own. The keys are relative to this input. + */ + std::unique_ptr locks; + + /** + * The source path of this input, if it has been fetched. + */ + mutable Sync> sourcePath; + + Lock(FlakeRef originalRef, FlakeRef lockedRef) + : originalRef(std::move(originalRef)) + , lockedRef(std::move(lockedRef)) + { + } + + Lock(const fetchers::Settings & fetchSettings, const nlohmann::json & json) + : originalRef(getFlakeRef(fetchSettings, json, "original")) + , lockedRef(getFlakeRef(fetchSettings, json, "locked")) + { + if (!lockedRef.input.isLocked(fetchSettings)) { + if (lockedRef.input.getNarHash()) + warn( + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", + lockedRef.to_string()); + else + throw Error( + "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); + } + + // For backward compatibility, lock file entries are implicitly final. + assert(!lockedRef.input.attrs.contains("__final")); + lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); + + if (auto locks = json.find("locks"); locks != json.end()) + this->locks = std::make_unique(fetchSettings, *locks); + } + + /** + * Return a deep copy of this lock, without the `sourcePath` + * cache. (An explicit method since `Sync` is not copyable.) + */ + Lock clone() const + { + Lock lock(originalRef, lockedRef); + if (locks) + lock.locks = std::make_unique(locks->clone()); + return lock; + } + + nlohmann::json toJSON() const + { + nlohmann::json n; + n["original"] = fetchers::attrsToJSON(originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); + assert(lockedRef.input.isFinal()); + if (locks) + n["locks"] = locks->toLocksJSON(); + return n; + } + }; + + /** + * The locks, keyed by slash-separated input attribute paths. A + * single-element key denotes an immediate input of the flake; + * longer keys denote overrides of transitive inputs. + */ + std::map locks; + + LockFileV8() = default; + + /** + * Construct from a `locks` attribute of a lock file. + */ + LockFileV8(const fetchers::Settings & fetchSettings, const nlohmann::json & locksJson) + { + for (auto & i : locksJson.items()) { + auto path = NonEmptyInputAttrPath::parse(i.key()); + if (!path) + throw Error("lock file contains an empty input attribute path"); + locks.emplace(std::move(*path), Lock(fetchSettings, i.value())); + } + } + + /** + * Construct from the JSON contents of a lock file. + */ + LockFileV8(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) + { + auto version = json.value("version", 0); + if (version != 8) + throw Error("lock file '%s' has unsupported version %d", path, version); + + if (auto locks = json.find("locks"); locks != json.end()) + *this = LockFileV8(fetchSettings, *locks); + } + + LockFileV8 clone() const + { + LockFileV8 res; + for (auto & [path, lock] : locks) + res.locks.emplace(path, lock.clone()); + return res; + } + + nlohmann::json toLocksJSON() const + { + auto res = nlohmann::json::object(); + for (auto & [path, lock] : locks) + res[printInputAttrPath(path)] = lock.toJSON(); + return res; + } + + nlohmann::json toJSON() const + { + nlohmann::json json; + json["version"] = 8; + json["locks"] = toLocksJSON(); + return json; + } + + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const + { + /* Return whether the input is either locked, or, if + `allow-dirty-locks` is enabled, it has a NAR hash. In the + latter case, we can verify the input but we may not be able to + fetch it from anywhere. */ + auto isConsideredLocked = [&](const fetchers::Input & input) { + return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); + }; + + for (auto & [path, lock] : locks) { + if (!isConsideredLocked(lock.lockedRef.input) || !lock.lockedRef.input.isFinal()) + return lock.lockedRef; + if (lock.locks) + if (auto ref = lock.locks->isUnlocked(fetchSettings)) + return ref; + } + + return std::nullopt; + } + + /** + * Flatten this lock file into a map from absolute input attribute + * paths to lock entries. Inline locks appear under the path of + * their containing entry. A colliding top-level (override) entry + * shadows an inline entry, matching the precedence of overrides + * at evaluation time. + */ + void + getAllLockEntries(std::map & res, const InputAttrPath & prefix = {}) const + { + for (auto & [path, lock] : locks) { + InputAttrPath absPath(prefix); + absPath.insert(absPath.end(), path.get().begin(), path.get().end()); + if (lock.locks) + lock.locks->getAllLockEntries(res, absPath); + /* Note: this shadows any colliding inline entry, since + the entry for the containing input sorts before the + override path and thus has been recursed into + already. */ + res.insert_or_assign(std::move(absPath), lock.lockedRef); + } + } +}; + +struct LockedFlakeV8 : LockedFlake +{ + /** + * The lock file in the sparse format (version 8). + */ + LockFileV8 lockFile; + + LockedFlakeV8(Flake && flake, LockFileV8 && lockFile) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + { + } + + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV8() : LockFileV8(fetchSettings, json, path)) + { + } + + [[noreturn]] static void notImplemented(std::string_view what) + { + throw Error("'%s' is not implemented yet for lock file version 8", what); + } + + std::map> getInputTargets(const InputAttrPath & prefix) const override + { + notImplemented("getInputTargets"); + } + + std::optional findInput(const InputAttrPath & path) const override + { + notImplemented("findInput"); + } + + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override + { + notImplemented("getSourcePath"); + } + + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override + { + return lockFile.isUnlocked(fetchSettings); + } + + unsigned int version() const override + { + return 8; + } + + std::map getAllLockEntries(bool fetchTransitive) const override + { + if (fetchTransitive) + throw Error("fetching transitive lock files is not implemented yet for lock file version 8"); + + std::map res; + lockFile.getAllLockEntries(res); + return res; + } + + nlohmann::json toJSON() const override + { + return lockFile.toJSON(); + } +}; + +std::unique_ptr parseLockFileV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) +{ + return std::make_unique(fetchSettings, std::move(flake), json, path); +} + +LockFlakeResult lockFlakeV8( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile) +{ + // FIXME: implement the version 8 lock algorithm. + throw Error("creating version 8 lock files is not implemented yet"); +} + +} // namespace nix::flake diff --git a/src/libflake/meson.build b/src/libflake/meson.build index a7dee62b306c..8db1aaa858fe 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -47,6 +47,7 @@ sources = files( 'flakeref.cc', 'input-attr-path.cc', 'lockfile-v7.cc', + 'lockfile-v8.cc', 'provenance.cc', 'settings.cc', 'url-name.cc', From d050b86ad4af2ec1568338971bc15825ac87be7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 12:39:54 +0200 Subject: [PATCH 22/26] libflake: Implement the version 8 lock algorithm lockFlakeV8() locks the immediate inputs of the flake, plus any overrides of transitive inputs declared in its flake.nix (stored as slash-separated paths like "foo/nixpkgs"), reusing entries from the old lock file when the flakeref is unchanged. 'follows' and relative path inputs are not stored; the latter are required to have a lock file of their own. Inputs that don't have a lock file get their transitive inputs locked inline in a recursive `locks` attribute, which is pruned when the input gains a lock file on a later update. Command line overrides are applied by merging them into the top-level flake's input tree, so they're handled uniformly with overrides declared in flake.nix (with the outermost override winning). Assisted-by: Claude Fable 5 --- src/libflake/lockfile-v8.cc | 261 +++++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 5 deletions(-) diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc index ba5665d5b56c..b5e850ae4283 100644 --- a/src/libflake/lockfile-v8.cc +++ b/src/libflake/lockfile-v8.cc @@ -258,12 +258,13 @@ struct LockedFlakeV8 : LockedFlake throw Error("'%s' is not implemented yet for lock file version 8", what); } - std::map> getInputTargets(const InputAttrPath & prefix) const override + std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const override { notImplemented("getInputTargets"); } - std::optional findInput(const InputAttrPath & path) const override + std::optional findInput(EvalState & state, const InputAttrPath & path) const override { notImplemented("findInput"); } @@ -310,10 +311,260 @@ LockFlakeResult lockFlakeV8( EvalState & state, const LockFlags & lockFlags, Flake flake, - const LockedFlake & oldLockFile) + const LockedFlake & _oldLockFile) { - // FIXME: implement the version 8 lock algorithm. - throw Error("creating version 8 lock files is not implemented yet"); + /* The old lock file to reuse entries from. Null if the old lock + file is not a version 8 lock file (e.g. when migrating from + version 7), or if we're relocking from scratch. Note that + updating all inputs (`inputUpdates` = nullopt) ignores the old + lock file, but lock file entries can then still be *copied* + from dependencies' own lock files. */ + const LockFileV8 * oldLockFile = nullptr; + if (auto old = dynamic_cast(&_oldLockFile); + old && !lockFlags.recreateLockFile && lockFlags.inputUpdates) + oldLockFile = &old->lockFile; + + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; + + std::set overridesUsed; + std::set updatesUsed; + std::set explicitCliOverrides; + + /* Apply command line overrides as if they were overrides declared + by the top-level flake (`inputs.foo.inputs.bar.url = ...`), + creating intermediate override entries as needed. They + overwrite any conflicting override in `flake.nix` ("outermost + override wins"). Note: overrides of inputs that don't exist at + the top level are left unapplied so the caller can warn about + them. */ + for (auto & [path, ref] : lockFlags.inputOverrides) { + auto input = get(flake.inputs, path.get().front()); + if (!input) + continue; + for (auto & elem : std::views::drop(path.get(), 1)) + input = &input->overrides[elem]; + input->ref = ref; + input->follows = std::nullopt; + overridesUsed.insert(path); + explicitCliOverrides.insert(path); + } + + LockFileV8 newLockFile; + + std::vector parents; + + std::function + computeLocks; + + computeLocks = [&]( + /* The declared inputs of the flake being + locked (from its flake.nix). */ + const FlakeInputs & flakeInputs, + /* The lock file being computed for this + flake. */ + LockFileV8 & output, + /* The absolute input attribute path of this + flake (empty for the top-level flake). */ + const InputAttrPath & absPrefix, + /* The old locks, if any, from which locks can + be copied. */ + const LockFileV8 * oldLocks, + /* The path of this flake's `flake.nix`. */ + const SourcePath & sourcePath) { + debug("computing lock file entries for '%s'", printInputAttrPath(absPrefix)); + + /* Compute the lock for a single input or override declared by + this flake. `relPath` is relative to this flake. Returns + std::nullopt for inputs that are not stored in the lock + file ('follows' and relative path inputs). */ + auto createLock = [&](const NonEmptyInputAttrPath & relPath, + const FlakeInput & input) -> std::optional { + InputAttrPath absPath(absPrefix); + absPath.insert(absPath.end(), relPath.get().begin(), relPath.get().end()); + auto nonEmptyAbsPath = *NonEmptyInputAttrPath::make(absPath); + auto absPathS = printInputAttrPath(absPath); + debug("computing input '%s'", absPathS); + + try { + updatesUsed.insert(absPath); + + if (input.follows) { + /* 'follows' inputs are not stored in the lock + file; they are resolved at evaluation time from + the flake.nix files. */ + if (!input.overrides.empty()) + throw Error( + "input '%s' has both 'follows' and overrides for its inputs, which is not supported by lock file version 8", + absPathS); + return std::nullopt; + } + + auto ref = input.ref.value_or( + FlakeRef::fromAttrs( + state.fetchSettings, {{"type", "indirect"}, {"id", std::string(relPath.inputName())}})); + + if (auto relativePath = ref.input.isRelative()) { + /* Relative path inputs (e.g. 'path:./foo') are + not stored in the lock file, since they change + along with the flake that declares them. If + they're flakes, they must have a lock file of + their own, which is used at evaluation time. */ + SourcePath resolved{ + sourcePath.accessor, CanonPath(relativePath->string(), sourcePath.path.parent().value())}; + if (input.isFlake && !(resolved / "flake.lock").pathExists()) + throw Error( + "relative path input '%s' does not have a lock file; run 'nix flake lock %s' to create it", + absPathS, + resolved); + return std::nullopt; + } + + auto explicitUpdate = lockFlags.inputUpdates && lockFlags.inputUpdates->count(nonEmptyAbsPath); + + auto oldLock = oldLocks ? get(oldLocks->locks, relPath) : nullptr; + + if (oldLock && !explicitUpdate && oldLock->originalRef.canonicalize() == ref.canonicalize()) { + /* Copy the input from the old lock file since its + flakeref didn't change. */ + + /* Check whether an explicit update of an input + *below* this one is requested. */ + bool mustRefetch = false; + if (lockFlags.inputUpdates) { + auto lb = lockFlags.inputUpdates->lower_bound(nonEmptyAbsPath); + mustRefetch = lb != lockFlags.inputUpdates->end() && lb->get().size() > absPath.size() + && std::equal(absPath.begin(), absPath.end(), lb->get().begin()); + } + + /* If so, and this input's transitive inputs are + locked here (because it has no lock file of its + own), refetch it and recompute its inline + locks. Otherwise the update path doesn't match + anything we can update, and the caller will + warn about it. */ + if (!mustRefetch || !oldLock->locks) { + debug("keeping existing input '%s'", absPathS); + return oldLock->clone(); + } + + auto inputFlake = getFlake(state, oldLock->lockedRef, useRegistriesInputs, absPath, true); + + LockFileV8::Lock lock(oldLock->originalRef, oldLock->lockedRef); + *lock.sourcePath.lock() = inputFlake.path.parent(); + lock.locks = std::make_unique(); + computeLocks(inputFlake.inputs, *lock.locks, absPath, oldLock->locks.get(), inputFlake.path); + return lock; + } + + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", absPathS); + + if (!lockFlags.allowUnlocked && !ref.input.isLocked(state.fetchSettings)) + throw Error("cannot update unlocked flake input '%s' in pure mode", absPathS); + + auto useRegistriesInput = + explicitCliOverrides.contains(nonEmptyAbsPath) ? fetchers::UseRegistries::All : useRegistriesInputs; + + if (input.isFlake) { + auto inputFlake = getFlake(state, ref, useRegistriesInput, absPath, true); + + warnRegistry(absPath, ref, inputFlake.resolvedRef, flake.path); + + LockFileV8::Lock lock(ref, inputFlake.lockedRef); + *lock.sourcePath.lock() = inputFlake.path.parent(); + + /* If the input doesn't have a lock file of its + own, lock its transitive inputs here, in the + `locks` attribute of this entry. */ + if (!inputFlake.lockFilePath().pathExists()) { + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(ref); + Finally cleanup([&]() { parents.pop_back(); }); + + lock.locks = std::make_unique(); + computeLocks( + inputFlake.inputs, + *lock.locks, + absPath, + oldLock && oldLock->locks ? oldLock->locks.get() : nullptr, + inputFlake.path); + } + + return lock; + } else { + auto cachedInput = + state.inputCache->getAccessor(state.fetchSettings, *state.store, ref.input, useRegistriesInput); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), ref.subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), ref.subdir); + + warnRegistry(absPath, ref, resolvedRef, flake.path); + + /* Note: `mountInput()` adds a NAR hash to + `lockedRef.input` if it doesn't have one. */ + auto storePath = + state.storePath(state.mountInput(lockedRef.input, ref.input, cachedInput.accessor, true, true)); + + LockFileV8::Lock lock(ref, lockedRef); + *lock.sourcePath.lock() = storePath; + return lock; + } + + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", absPathS); + throw; + } + }; + + for (auto & [id, input] : flakeInputs) { + auto relPath = NonEmptyInputAttrPath::append({}, id); + + if (auto lock = createLock(relPath, input)) + output.locks.emplace(relPath, std::move(*lock)); + + /* Store the overrides declared by this flake for the + transitive inputs of this input + (e.g. `inputs.foo.inputs.bar.url = ...`), keyed by + their path relative to this flake. */ + [&](this const auto & recurse, const NonEmptyInputAttrPath & prefix, const FlakeInput & input) -> void { + for (auto & [id2, override] : input.overrides) { + auto relPath2 = NonEmptyInputAttrPath::append(prefix, id2); + if (override.follows) { + /* 'follows' overrides are not stored; they + are resolved at evaluation time from the + flake.nix files. */ + if (!override.overrides.empty()) + throw Error( + "input '%s' has both 'follows' and overrides for its inputs, which is not supported by lock file version 8", + printInputAttrPath(relPath2)); + continue; + } + if (override.ref) + if (auto lock = createLock(relPath2, override)) + output.locks.emplace(relPath2, std::move(*lock)); + recurse(relPath2, override); + } + }(relPath, input); + } + }; + + computeLocks(flake.inputs, newLockFile, {}, oldLockFile, flake.path); + + return { + .lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)), + .overridesUsed = std::move(overridesUsed), + .updatesUsed = std::move(updatesUsed), + }; } } // namespace nix::flake From 4061fb1259ba282cd9bf3cb41d4f45139d6aa56e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 12:39:54 +0200 Subject: [PATCH 23/26] lockFlake(): Dispatch on the lock file version The old lock file is parsed with the parser matching its version. The version of the new lock file is that of the existing lock file; only `--recreate-lock-file` (or the absence of a lock file) switches to the version configured by the `lock-file-format` setting. Version changes are shown in the lock file diff. lockFlakeV7() now ignores a non-v7 old lock file instead of throwing `std::bad_cast`, so migrating from version 8 back to 7 works. Assisted-by: Claude Fable 5 --- src/libflake/flake.cc | 32 ++++++++++++++++++++++++++++---- src/libflake/lockfile-v7.cc | 6 +++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index dfc2a48f9c66..0cccf128e58b 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -434,7 +434,9 @@ std::unique_ptr parseLockFile( { auto version = json.is_null() ? versionIfMissing : (unsigned int) json.value("version", 0); - if (version >= 5 && version <= 7) + if (version == 8) + return parseLockFileV8(fetchSettings, std::move(flake), json, path); + else if (version >= 5 && version <= 7) return parseLockFileV7(fetchSettings, std::move(flake), json, path); else throw Error("lock file '%s' has unsupported version %d", path, version); @@ -493,13 +495,35 @@ std::unique_ptr lockFlake( } } - // FIXME: dispatch on the lock file version here. - auto oldLockedFlake = parseLockFile(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath)); + std::optional oldVersion; + if (!oldLockFileJson.is_null()) { + oldVersion = oldLockFileJson.value("version", 0); + if (*oldVersion < 5 || *oldVersion > 8) + throw Error("lock file '%s' has unsupported version %d", lockFilePath, *oldVersion); + } + + /* Determine the version of the new lock file: the existing + lock file's version wins, unless `--recreate-lock-file` was + passed (or there is no lock file), in which case the + `lock-file-format` setting is used. */ + unsigned int version = oldVersion && !lockFlags.recreateLockFile ? (*oldVersion == 8 ? 8 : 7) + : (unsigned int) settings.lockFileFormat; + + if (version != 7 && version != 8) + throw Error("unsupported lock file format version %d; supported versions are 7 and 8", version); + + /* Parse the old lock file. If there is no lock file, get an + empty lock file of the version we're producing, so that the + lock functions below receive the type they expect. */ + auto oldLockedFlake = + parseLockFile(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath), version); debug("old lock file: %s", oldLockedFlake->to_string()); + auto lockFlakeForVersion = version == 8 ? lockFlakeV8 : lockFlakeV7; + auto [lockedFlake, overridesUsed, updatesUsed] = - lockFlakeV7(settings, state, lockFlags, std::move(flake), *oldLockedFlake); + lockFlakeForVersion(settings, state, lockFlags, std::move(flake), *oldLockedFlake); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc index f19a48740b9c..d10aa093d461 100644 --- a/src/libflake/lockfile-v7.cc +++ b/src/libflake/lockfile-v7.cc @@ -524,7 +524,11 @@ LockFlakeResult lockFlakeV7( Flake flake, const LockedFlake & _oldLockFile) { - auto & oldLockFile = dynamic_cast(_oldLockFile).lockFile; + /* If the old lock file is not a version 7 lock file (e.g. when + migrating from version 8), ignore it. */ + auto oldLockedFlake = dynamic_cast(&_oldLockFile); + LockFileV7 emptyLockFile; + auto & oldLockFile = oldLockedFlake ? oldLockedFlake->lockFile : emptyLockFile; auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; From 9f747f80bee40fd2d4857f24a94cc3e39df59458 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 18:51:14 +0200 Subject: [PATCH 24/26] lockfile-v8.cc: Clarify comment terminology Say "takes precedence over" instead of "shadows", and refer to the entries in the recursive `locks` attribute as "nested" entries rather than "inline" ones. Assisted-by: Claude Fable 5 --- src/libflake/lockfile-v8.cc | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc index b5e850ae4283..35ddf7510844 100644 --- a/src/libflake/lockfile-v8.cc +++ b/src/libflake/lockfile-v8.cc @@ -207,9 +207,10 @@ struct LockFileV8 /** * Flatten this lock file into a map from absolute input attribute - * paths to lock entries. Inline locks appear under the path of - * their containing entry. A colliding top-level (override) entry - * shadows an inline entry, matching the precedence of overrides + * paths to lock entries. Entries in the nested `locks` of an + * entry appear under the path of that entry. If a top-level + * (override) entry denotes the same path as a nested entry, the + * former takes precedence, matching the precedence of overrides * at evaluation time. */ void @@ -220,10 +221,10 @@ struct LockFileV8 absPath.insert(absPath.end(), path.get().begin(), path.get().end()); if (lock.locks) lock.locks->getAllLockEntries(res, absPath); - /* Note: this shadows any colliding inline entry, since - the entry for the containing input sorts before the - override path and thus has been recursed into - already. */ + /* Note: this takes precedence over any nested entry for + the same path, since the entry for the containing input + sorts before the override path and thus has been + recursed into already. */ res.insert_or_assign(std::move(absPath), lock.lockedRef); } } @@ -444,10 +445,10 @@ LockFlakeResult lockFlakeV8( /* If so, and this input's transitive inputs are locked here (because it has no lock file of its - own), refetch it and recompute its inline - locks. Otherwise the update path doesn't match - anything we can update, and the caller will - warn about it. */ + own), refetch it and recompute its nested + `locks`. Otherwise the update path doesn't + match anything we can update, and the caller + will warn about it. */ if (!mustRefetch || !oldLock->locks) { debug("keeping existing input '%s'", absPathS); return oldLock->clone(); From 431b73d17ecf81a2bd06d672bb66d4df7e4d3784 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Aug 2026 20:36:00 +0200 Subject: [PATCH 25/26] libflake: Implement evaluation-time resolution for version 8 lock files LockedFlakeV8 now implements getInputTargets(), findInput() and getSourcePath(), making `nix build`/`nix eval` etc. work on version 8 lock files. Since a sparse lock file only contains the locks of the immediate inputs of a flake (plus overrides and the nested `locks` of inputs that don't have a lock file of their own), the resolution of transitive inputs is *delegated*: for each input, a child `LockedFlake` is constructed lazily by fetching the input and reading its own lock file (which may use any lock file version - so the entire dependency graph of a version 7 lock file is honored for its subtree), or from the nested `locks` of its entry. Queries below an input are forwarded to its child, translating input attribute paths and 'follows' targets between the child's namespace and ours. Override entries in our lock file (slash-separated keys like "foo/nixpkgs") take precedence over the delegated resolution, and relative path inputs are resolved against our own source tree and delegated via their own lock file. Per the LockedFlake contract, query paths must be fully resolved: an error is thrown when the descent encounters an input declared as a 'follows'. Note that unlike version 7, which stores 'follows' in the lock file, we have to check the flake.nix declarations for this - without it, a path crossing one of our 'follows' overrides would be delegated to a child that doesn't know about the override. Note that a missing entry in a transitive lock file is a fatal evaluation error: unlike version 7, version 8 does not silently repair incomplete transitive lock files. (The top-level lock file is still completed by lockFlake() before evaluation, as before.) Assisted-by: Claude Fable 5 --- src/libflake/lockfile-v8.cc | 333 +++++++++++++++++++++++++++++++++++- 1 file changed, 328 insertions(+), 5 deletions(-) diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc index 35ddf7510844..bd0ee31f1fc9 100644 --- a/src/libflake/lockfile-v8.cc +++ b/src/libflake/lockfile-v8.cc @@ -228,8 +228,41 @@ struct LockFileV8 res.insert_or_assign(std::move(absPath), lock.lockedRef); } } + + /** + * Return the lock for the exact input attribute path `path`, if + * any. + */ + const Lock * findLock(const InputAttrPath & path) const + { + if (auto p = NonEmptyInputAttrPath::make(path)) { + auto i = locks.find(*p); + if (i != locks.end()) + return &i->second; + } + return nullptr; + } }; +/** + * Read and parse the lock file of `flake` (which may use any lock + * file version). The lock file must exist. + */ +static std::unique_ptr readLockFile(EvalState & state, Flake flake) +{ + auto lockFilePath = flake.lockFilePath(); + + nlohmann::json json; + + try { + json = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + + return parseLockFile(state.fetchSettings, std::move(flake), json, fmt("%s", lockFilePath)); +} + struct LockedFlakeV8 : LockedFlake { /** @@ -254,25 +287,315 @@ struct LockedFlakeV8 : LockedFlake { } - [[noreturn]] static void notImplemented(std::string_view what) + /** + * Cache of the child `LockedFlake`s to which we delegate the + * resolution of transitive inputs, keyed by the input attribute + * path of their lock file entry (or of a relative path input). + */ + mutable Sync>> children; + + /** + * Look up the `FlakeInput` declared by this flake's `flake.nix` + * for `path`: for a single-element path, the input declaration; + * for longer paths, an override + * (e.g. `inputs.foo.inputs.bar.url = ...`). + */ + const FlakeInput * lookupDeclaration(const InputAttrPath & path) const { - throw Error("'%s' is not implemented yet for lock file version 8", what); + const FlakeInputs * inputs = &flake.inputs; + const FlakeInput * input = nullptr; + for (auto & elem : path) { + input = get(*inputs, elem); + if (!input) + return nullptr; + inputs = &input->overrides; + } + return input; + } + + /** + * Return the input attribute path of the entry that governs the + * resolution below `path`: the longest prefix of `path` (a proper + * prefix if `proper` is set) that has a lock file entry, or the + * first element of `path` if it denotes a relative path input + * (which is not stored in the lock file). Throws an error if a + * prefix of `path` denotes an input that this flake declares as a + * "follows", since `path` is then not fully resolved (see + * `LockedFlake::resolveFollows()`); note that unlike version 7, + * which stores "follows" in the lock file, we have to check the + * `flake.nix` declarations for this. + */ + std::optional findGoverningKey(const InputAttrPath & path, bool proper) const + { + std::optional res; + + InputAttrPath prefix; + for (auto & elem : path) { + prefix.push_back(elem); + if (auto decl = lookupDeclaration(prefix); decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath(prefix)); + if ((!proper || prefix.size() < path.size()) && lockFile.findLock(prefix)) + res = prefix; + } + + if (res) + return res; + + auto decl = get(flake.inputs, path.front()); + if (decl && decl->ref && decl->ref->input.isRelative()) + return InputAttrPath{path.front()}; + + return std::nullopt; + } + + /** + * Return the child `LockedFlake` that governs the resolution of + * the transitive inputs of the input denoted by `key` (the path + * of an entry in our lock file, or of a relative path input). + */ + std::shared_ptr getChild(EvalState & state, const InputAttrPath & key) const + { + { + auto children_(children.lock()); + if (auto i = get(*children_, key)) + return *i; + } + + /* Note: we construct the child without holding the `children` + lock, so concurrent calls don't get serialized. Racing + constructions of the same child are harmless since they + produce equivalent children. */ + std::shared_ptr child; + + if (auto lock = lockFile.findLock(key)) { + auto inputFlake = getFlake(state, lock->lockedRef, fetchers::UseRegistries::No, {}, false); + + { + auto sourcePath(lock->sourcePath.lock()); + if (!*sourcePath) + *sourcePath = inputFlake.path.parent(); + } + + if (lock->locks) + /* The input doesn't have a lock file of its own; its + transitive inputs are locked in the nested `locks` + of its entry. */ + child = std::make_shared(std::move(inputFlake), lock->locks->clone()); + else if (inputFlake.lockFilePath().pathExists()) + child = readLockFile(state, std::move(inputFlake)); + else + throw Error( + "flake input '%s' does not have a lock file, and its transitive inputs are not locked in the lock file of flake '%s'", + printInputAttrPath(key), + flake.lockedRef); + } else { + /* A relative path input (e.g. 'path:./foo'), resolved + against our own source tree. It is required to have a + lock file of its own. */ + auto decl = lookupDeclaration(key); + assert(decl && decl->ref); + auto relativePath = decl->ref->input.isRelative(); + assert(relativePath); + SourcePath resolvedPath{ + flake.path.accessor, CanonPath(relativePath->string(), flake.path.path.parent().value())}; + auto inputFlake = readFlake(state, *decl->ref, *decl->ref, *decl->ref, resolvedPath, {}); + if (!inputFlake.lockFilePath().pathExists()) + throw Error( + "relative path input '%s' does not have a lock file; run 'nix flake lock %s' to create it", + printInputAttrPath(key), + resolvedPath); + child = readLockFile(state, std::move(inputFlake)); + } + + auto children_(children.lock()); + return children_->emplace(key, std::move(child)).first->second; } std::map> getInputTargets(EvalState & state, const InputAttrPath & prefix) const override { - notImplemented("getInputTargets"); + std::map> res; + + if (prefix.empty()) { + for (auto & [name, input] : flake.inputs) + res.emplace(name, input.follows); + return res; + } + + /* Delegate to the child that governs `prefix`. */ + auto key = findGoverningKey(prefix, false); + if (!key) + throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); + + auto child = getChild(state, *key); + InputAttrPath rest(prefix.begin() + key->size(), prefix.end()); + + for (auto & [name, target] : child->getInputTargets(state, rest)) { + if (target) { + /* Translate the target from child-relative to our + root. */ + InputAttrPath absTarget(*key); + absTarget.insert(absTarget.end(), target->begin(), target->end()); + res.emplace(name, std::move(absTarget)); + } else + res.emplace(name, std::nullopt); + } + + /* Apply the 'follows' overrides declared by this flake for + the inputs of `prefix`; they take precedence over whatever + the child declares. */ + for (auto & [name, target] : res) { + InputAttrPath path(prefix); + path.push_back(name); + auto decl = lookupDeclaration(path); + if (decl && decl->follows) + target = *decl->follows; + } + + return res; } std::optional findInput(EvalState & state, const InputAttrPath & path) const override { - notImplemented("findInput"); + if (path.empty()) + return std::nullopt; + + auto lock = lockFile.findLock(path); + + if (path.size() == 1) { + auto decl = get(flake.inputs, path.front()); + + if (decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath(path)); + + if (!decl) { + if (!lock) + return std::nullopt; + /* A lock file entry without a corresponding + declaration in flake.nix. */ + return InputInfo{.lockedRef = lock->lockedRef}; + } + + if (decl->ref && decl->ref->input.isRelative()) + /* Relative path inputs are not stored in the lock + file. */ + return InputInfo{ + .lockedRef = *decl->ref, + .isFlake = decl->isFlake, + .buildTime = decl->buildTime, + .parentInputAttrPath = InputAttrPath{}}; + + if (!lock) + throw Error( + "lock file of flake '%s' does not contain an entry for input '%s'", flake.lockedRef, path.front()); + + return InputInfo{.lockedRef = lock->lockedRef, .isFlake = decl->isFlake, .buildTime = decl->buildTime}; + } + + /* Delegate to the child that governs this path. */ + auto key = findGoverningKey(path, true); + if (!key) { + if (!lock) + return std::nullopt; + /* An override entry below an input that we know nothing + about. */ + return InputInfo{.lockedRef = lock->lockedRef}; + } + + auto child = getChild(state, *key); + InputAttrPath rest(path.begin() + key->size(), path.end()); + + if (lock) { + /* An override entry in our lock file takes precedence + over the child's resolution; we only consult the child + for the input's metadata (such as whether it's a + flake). */ + std::optional childInfo; + try { + childInfo = child->findInput(state, rest); + } catch (Error &) { + } + return InputInfo{ + .lockedRef = lock->lockedRef, + .isFlake = !childInfo || childInfo->isFlake, + .buildTime = childInfo && childInfo->buildTime, + }; + } + + auto info = child->findInput(state, rest); + + if (info && info->parentInputAttrPath) { + /* Translate the parent path from child-relative to our + root. */ + InputAttrPath parent(*key); + parent.insert(parent.end(), info->parentInputAttrPath->begin(), info->parentInputAttrPath->end()); + info->parentInputAttrPath = std::move(parent); + } + + return info; } SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override { - notImplemented("getSourcePath"); + if (inputAttrPath.empty()) + /* The root flake. */ + return flake.path.parent(); + + if (auto lock = lockFile.findLock(inputAttrPath)) { + { + auto sourcePath(lock->sourcePath.lock()); + if (*sourcePath) + return **sourcePath; + } + + /* Note: we fetch without holding the `sourcePath` lock, + so concurrent calls don't get serialized. Racing + fetches of the same input are harmless since they + produce the same path. */ + /* Note: `lockedRef` is a copy since `mountInput()` may + modify the input (e.g. adding a `narHash` + attribute). */ + auto lockedRef = lock->lockedRef; + auto accessor = + state.inputCache + ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) + .accessor; + auto res = state.storePath(state.mountInput(lockedRef.input, lock->lockedRef.input, accessor, true, true)) + / CanonPath(lockedRef.subdir); + + *lock->sourcePath.lock() = res; + + return res; + } + + if (inputAttrPath.size() == 1) { + auto decl = get(flake.inputs, inputAttrPath.front()); + if (decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(inputAttrPath)); + if (decl && decl->ref && decl->ref->input.isRelative()) { + /* Resolve relative path inputs against our own source + tree. */ + auto parentPath = flake.path.parent(); + return {parentPath.accessor, CanonPath(decl->ref->input.isRelative()->string(), parentPath.path)}; + } + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + } + + auto key = findGoverningKey(inputAttrPath, true); + if (!key) + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + + InputAttrPath rest(inputAttrPath.begin() + key->size(), inputAttrPath.end()); + return getChild(state, *key)->getSourcePath(state, rest); } std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override From 7c555a949cd7d67e3c9d82a69406ee7228c2b8a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 22:18:49 +0200 Subject: [PATCH 26/26] Add the lock-file-v8 experimental feature Reading or creating version 8 (sparse) lock files now requires the `lock-file-v8` experimental feature. The checks are in parseLockFileV8() and lockFlakeV8(), so they also cover transitive version 8 lock files encountered during evaluation and commands like `nix flake diff-locks`. Version 7 lock files are unaffected. Assisted-by: Claude Fable 5 --- src/libflake/include/nix/flake/settings.hh | 3 ++- src/libflake/lockfile-v8.cc | 6 ++++++ src/libutil/experimental-features.cc | 13 ++++++++++++- .../include/nix/util/experimental-features.hh | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 4a9f264d8aee..d5e5566cfa6d 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -49,7 +49,8 @@ struct Settings : public Config R"( The lock file format version to use when creating a new lock file (7 or 8). An existing lock file keeps its version - unless `--recreate-lock-file` is passed. + unless `--recreate-lock-file` is passed. Note: version 8 + requires the `lock-file-v8` experimental feature. )", {}, true}; diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc index bd0ee31f1fc9..d0b6abb988e2 100644 --- a/src/libflake/lockfile-v8.cc +++ b/src/libflake/lockfile-v8.cc @@ -25,7 +25,9 @@ #include "nix/flake/flakeref.hh" #include "nix/util/ansicolor.hh" #include "nix/util/canon-path.hh" +#include "nix/util/configuration.hh" #include "nix/util/error.hh" +#include "nix/util/experimental-features.hh" #include "nix/util/finally.hh" #include "nix/util/fmt.hh" #include "nix/util/logging.hh" @@ -627,6 +629,8 @@ struct LockedFlakeV8 : LockedFlake std::unique_ptr parseLockFileV8( const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) { + experimentalFeatureSettings.require(Xp::LockFileV8); + return std::make_unique(fetchSettings, std::move(flake), json, path); } @@ -637,6 +641,8 @@ LockFlakeResult lockFlakeV8( Flake flake, const LockedFlake & _oldLockFile) { + experimentalFeatureSettings.require(Xp::LockFileV8); + /* The old lock file to reuse entries from. Null if the old lock file is not a version 8 lock file (e.g. when migrating from version 7), or if we're relocking from scratch. Note that diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 674deec7f85c..cad83f70d4b4 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -27,7 +27,7 @@ void MissingExperimentalFeature::anchor() {} * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::CNSA); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::LockFileV8); constexpr std::array xpFeatureDetails = {{ { @@ -317,6 +317,17 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::LockFileV8, + .name = "lock-file-v8", + .description = R"( + Enable support for version 8 ("sparse") flake lock files, which + only store the locks of a flake's immediate inputs. Use the + [`lock-file-format`](@docroot@/command-ref/conf-file.md#conf-lock-file-format) + setting to create version 8 lock files. + )", + .trackingUrl = "", + }, }}; static_assert( diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index f20024f0387e..4eed26bcfb88 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -42,6 +42,7 @@ enum struct ExperimentalFeature { WasmDerivations, Provenance, CNSA, + LockFileV8, }; extern std::set stabilizedFeatures;