diff --git a/CLAUDE.md b/CLAUDE.md index 27268bf11f5f..6c5caf64af6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,5 @@ Observe the following rules when contributing to this repository: -* Before committing, run ./maintainers/format.sh to detect/fix any formatting issues. +* Before committing, run `./maintainers/format.sh` to detect/fix any formatting issues. If you've only touched C++ files, run `run ./maintainers/format.sh clang-format` since it's a lot faster. * Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits. diff --git a/maintainers/format.sh b/maintainers/format.sh index b2902e6dc6c8..c15d47a7faab 100755 --- a/maintainers/format.sh +++ b/maintainers/format.sh @@ -9,7 +9,14 @@ if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then exit 1; fi; -while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files; do +# The argument is either `--until-stable` or the ID of the single +# hook to run. +hook="" +if [ "${1:-}" != "--until-stable" ]; then + hook="${1:-}" +fi + +while ! pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files ${hook:+"$hook"}; do if [ "${1:-}" != "--until-stable" ]; then exit 1 fi diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index c8e2992be7e2..1179d9679434 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -10,7 +10,7 @@ namespace nix::flake_schemas { using namespace eval_cache; using namespace flake; -static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state) +static std::unique_ptr getBuiltinDefaultSchemasFlake(EvalState & state) { auto accessor = make_ref(); @@ -49,11 +49,11 @@ ref call( #include "call-flake-schemas.nix.gen.hh" ; - auto lockedDefaultSchemasFlake = defaultSchemasFlake - ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {}) - : getBuiltinDefaultSchemasFlake(state); + std::shared_ptr lockedDefaultSchemasFlake = + defaultSchemasFlake ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {}) + : getBuiltinDefaultSchemasFlake(state); auto lockedDefaultSchemasFlakeFingerprint = - lockedDefaultSchemasFlake.getFingerprint(*state.store, state.fetchSettings); + lockedDefaultSchemasFlake->getFingerprint(*state.store, state.fetchSettings); std::optional fingerprint2; if (allowEvalCache && evalSettings.useEvalCache && evalSettings.pureEval && fingerprint @@ -78,7 +78,7 @@ ref call( state.parseExprFromString(callFlakeSchemasNix, state.rootPath(CanonPath::root)), *vCallFlakeSchemas); auto vFlake = state.allocValue(); - flake::callFlake(state, *lockedFlake, *vFlake); + flake::callFlake(state, lockedFlake, *vFlake); auto vDefaultSchemasFlake = state.allocValue(); if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas"))) diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index c43b4f87ed43..2d47a9d82248 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -6,7 +6,6 @@ #include "nix/cmd/common-eval-args.hh" #include "nix/store/path.hh" #include "nix/store/store-reference.hh" -#include "nix/flake/lockfile.hh" #include diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d2022aeb09d0..ed646345c1d7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -310,7 +310,8 @@ ref InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = make_ref(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = + std::shared_ptr(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } // _lockedFlake is now non-null but still just a shared_ptr return ref(_lockedFlake); @@ -328,12 +329,10 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const { auto lockedFlake = getLockedFlake(); - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { - if (lockedNode->isFlake) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } + if (auto nixpkgsInput = lockedFlake->findInput(*state, lockedFlake->resolveFollows(*state, {"nixpkgs"}))) { + if (nixpkgsInput->isFlake) { + debug("using nixpkgs flake '%s'", nixpkgsInput->lockedRef); + return std::move(nixpkgsInput->lockedRef); } } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2285e0c3cf51..bc01a7d5b0e5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -48,19 +48,9 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = R"( - Recreate the flake's lock file from scratch. - - > **DEPRECATED** - > - > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. - )", + .description = "Recreate the flake's lock file from scratch.", .category = category, - .handler = {[&]() { - lockFlags.recreateLockFile = true; - warn( - "'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); - }}, + .handler = {&lockFlags.recreateLockFile, true}, }); addFlag({ @@ -117,7 +107,7 @@ MixFlakeOptions::MixFlakeOptions() if (!path) throw UsageError( "--update-input was passed a zero-length input path, which would refer to the flake itself, not an input"); - lockFlags.inputUpdates.insert(*path); + lockFlags.inputUpdates->insert(*path); }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); @@ -175,23 +165,24 @@ MixFlakeOptions::MixFlakeOptions() .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); - auto flake = flake::lockFlake( + auto lockedFlake = flake::lockFlake( flakeSettings, *evalState, parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()), {.writeLockFile = false}); - for (auto & [inputName, input] : flake.lockFile.root->inputs) { - auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes - if (auto input3 = std::dynamic_pointer_cast(input2)) { + + for (auto & inputName : lockedFlake->getInputNames(*evalState, {})) { + if (auto input = + lockedFlake->findInput(*evalState, lockedFlake->resolveFollows(*evalState, {inputName}))) { fetchers::Attrs extraAttrs; - if (!input3->lockedRef.subdir.empty()) { - extraAttrs["dir"] = input3->lockedRef.subdir; + if (!input->lockedRef.subdir.empty()) { + extraAttrs["dir"] = input->lockedRef.subdir; } overrideRegistry( fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}), - input3->lockedRef.input, + input->lockedRef.input, extraAttrs); } } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 6eaab847a0a6..a14a6b2cf813 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -20,7 +20,6 @@ #include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" #include "nix/flake/flake.hh" -#include "nix/flake/lockfile.hh" #include "nix/util/users.hh" #include "nix/cmd/editor-for.hh" #include "nix/util/finally.hh" diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 2558236a7e5a..2aa5aa333e48 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -185,9 +185,10 @@ nix_locked_flake * nix_flake_lock( nix_clear_err(context); try { eval_state->state.resetFileCache(); - auto lockedFlake = nix::make_ref(nix::flake::lockFlake( - *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); - return new nix_locked_flake{lockedFlake}; + std::shared_ptr lockedFlake( + nix::flake::lockFlake( + *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags)); + return new nix_locked_flake{nix::ref(lockedFlake)}; } NIXC_CATCH_ERRS_NULL } @@ -203,7 +204,7 @@ nix_value * nix_locked_flake_get_output_attrs( nix_clear_err(context); try { auto v = nix_alloc_value(context, evalState); - nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value); + nix::flake::callFlake(evalState->state, lockedFlake->lockedFlake.get_ptr(), *v->value); return v; } NIXC_CATCH_ERRS_NULL diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index d1037efcd23b..6b1b061550cf 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -1,79 +1,64 @@ # This is a helper to callFlake() to lazily fetch flake inputs. -# The contents of the lock file, in JSON format. -lockFileStr: +# An external value wrapping the C++ `LockedFlake` object. +lockedFlake: -# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets, -# with sourceInfo.outPath providing an SourceAccessor to a previously -# fetched tree. This is necessary for possibly unlocked inputs, in -# particular the root input, but also --override-inputs pointing to -# unlocked trees. -overrides: +# A primop that, given the locked flake and the input attribute path +# of an input, returns an attribute set mapping the names of its +# inputs to either null (for a regular input) or the input attribute +# path of the target of a "follows" input. +listFlakeInputs: + +# A primop that, given the locked flake and the input attribute path +# of an input, fetches that input and returns an attribute set +# describing it. +fetchFlakeInput: let - inherit (builtins) mapAttrs; - - lockFile = builtins.fromJSON lockFileStr; - - # Resolve a input spec into a node name. An input spec is - # either a node name, or a 'follows' path from the root - # node. - resolveInput = - inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec; - - # Follow an input attrpath (e.g. ["dwarffs" "nixpkgs"]) from the - # root node, returning the final node. - getInputByPath = - nodeName: path: - if path == [ ] then - nodeName - else - getInputByPath - # Since this could be a 'follows' input, call resolveInput. - (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) - (builtins.tail path); - - allNodes = mapAttrs ( - key: node: + inherit (builtins) mapAttrs foldl'; + + # Construct the input denoted by the input attribute path + # `inputAttrPath` (where `[ ]` denotes the top-level flake). This returns `edges` + # (mapping each input name of this input to the input it denotes, + # following "follows" indirections) and `result` (the value of this + # input, i.e. what ends up in the `inputs` attribute of a flake). + mkInput = + inputAttrPath: let - hasOverride = overrides ? ${key}; - isRelative = node.locked.type or null == "path" && builtins.substring 0 1 node.locked.path != "/"; - - parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + info = fetchFlakeInput lockedFlake inputAttrPath; sourceInfo = - if node.buildTime or false then + if info.buildTime then derivation { name = "source"; builder = "builtin:fetch-tree"; system = "builtin"; __structuredAttrs = true; - input = node.locked; + input = info.locked; outputHashMode = "recursive"; - outputHash = node.locked.narHash; + outputHash = info.locked.narHash; } - else if hasOverride then - overrides.${key}.sourceInfo - else if isRelative then - parentNode.sourceInfo else - # FIXME: remove obsolete node.info. - # Note: lock file entries are always final. - builtins.fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]); + info.sourceInfo; - subdir = overrides.${key}.dir or node.locked.dir or ""; + subdir = if info.buildTime then info.locked.dir or "" else info.dir; - outPath = - if !hasOverride && isRelative then - parentNode.outPath + (if node.locked.path == "" then "" else "/" + node.locked.path) - else - sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir); + outPath = sourceInfo.outPath + (if subdir == "" then "" else "/" + subdir); flake = import (outPath + "/flake.nix"); - inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) ( - node.inputs or { } - ); + # Note: constructing `edges` only consults the lock data (via + # `listFlakeInputs`), so it never causes anything to be + # fetched. A regular input is constructed in place; a "follows" + # input is resolved by walking the edges from the top-level + # flake, so every distinct input is constructed (and evaluated) + # only once. + edges = mapAttrs ( + name: target: + if target == null then mkInput (inputAttrPath ++ [ name ]) else getInputByAttrPath target + ) (listFlakeInputs lockedFlake inputAttrPath); + + inputs = mapAttrs (name: input: input.result) edges; outputs = flake.outputs (inputs // { self = result; }); @@ -97,17 +82,22 @@ let in { + inherit edges; + result = - if node.flake or true then + if info.flake then assert builtins.isFunction flake.outputs; - assert !(node.buildTime or false); + assert !info.buildTime; result else sourceInfo // { inherit sourceInfo outPath; }; + }; + + # Follow an input attribute path (e.g. ["dwarffs" "nixpkgs"]) from + # the top-level flake, returning the final input. + getInputByAttrPath = inputAttrPath: foldl' (input: name: input.edges.${name}) root inputAttrPath; - inherit outPath sourceInfo; - } - ) lockFile.nodes; + root = mkInput [ ]; in -allNodes.${lockFile.root}.result +root.result diff --git a/src/libflake/diff.cc b/src/libflake/diff.cc new file mode 100644 index 000000000000..6e585a1ec6e3 --- /dev/null +++ b/src/libflake/diff.cc @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include + +#include "nix/flake/flake.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/flake/flakeref.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/fmt.hh" + +namespace nix::flake { + +static std::string describe(const FlakeRef & flakeRef) +{ + auto s = fmt("'%s'", flakeRef.to_string(true)); + + if (auto lastModified = flakeRef.input.getLastModified()) + s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); + + return s; +} + +static std::string describe(const LockedFlake::LockEntry & entry) +{ + if (auto lockedRef = std::get_if(&entry)) + return describe(*lockedRef); + else + return fmt("follows '%s'", printInputAttrPath(std::get(entry))); +} + +std::string +diffLockedFlakes(const LockedFlake & oldLockedFlake, const LockedFlake & newLockedFlake, bool fetchTransitive) +{ + std::string res; + + if (oldLockedFlake.version() != newLockedFlake.version()) + res += + fmt("• " ANSI_BOLD "Updated lock file version from %d to %d" ANSI_NORMAL "\n", + oldLockedFlake.version(), + newLockedFlake.version()); + + auto oldFlat = oldLockedFlake.getAllLockEntries(fetchTransitive); + auto newFlat = newLockedFlake.getAllLockEntries(fetchTransitive); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += + fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", + printInputAttrPath(j->first), + describe(j->second)); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); + ++i; + } else { + if (i->second != j->second) { + res += + fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", + printInputAttrPath(i->first), + describe(i->second), + describe(j->second)); + } + ++i; + ++j; + } + } + + return res; +} + +} // namespace nix::flake diff --git a/src/libflake/flake-impl.hh b/src/libflake/flake-impl.hh new file mode 100644 index 000000000000..52a1469a8a65 --- /dev/null +++ b/src/libflake/flake-impl.hh @@ -0,0 +1,25 @@ +#pragma once +///@file +/// Internal declarations shared between the lock file +/// implementations. Not part of the public libflake API. + +#include "nix/flake/flakeref.hh" +#include "nix/flake/input-attr-path.hh" +#include "nix/util/source-path.hh" + +namespace nix::flake { + +/** + * Warn against the use of indirect flakerefs (but only for top-level + * inputs, since we don't want to annoy users about flakes that are + * not under their control). `ref` is the input's flakeref as declared + * and `resolvedRef` its registry-resolved counterpart; `topFlakePath` + * is the `flake.nix` of the top-level flake. + */ +void warnRegistry( + const InputAttrPath & inputAttrPath, + const FlakeRef & ref, + const FlakeRef & resolvedRef, + const SourcePath & topFlakePath); + +} // namespace nix::flake diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 0ba33ac326f0..0cccf128e58b 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -22,10 +23,10 @@ #include "nix/util/ref.hh" #include "nix/util/environment-variables.hh" #include "nix/flake/flake.hh" +#include "flake-impl.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-cache.hh" #include "nix/expr/eval-settings.hh" -#include "nix/flake/lockfile.hh" #include "nix/expr/eval-inline.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" @@ -40,6 +41,9 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" #include "nix/expr/fetch-tree.hh" +#include "nix/expr/json-to-value.hh" +#include "nix/expr/primops.hh" +#include "nix/expr/print.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -59,6 +63,7 @@ #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" #include "nix/util/source-path.hh" +#include "nix/util/strings.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" @@ -373,7 +378,7 @@ static FlakeRef applySelfAttrs(const FlakeRef & ref, const Flake & flake) return newRef; } -static Flake getFlake( +Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, @@ -420,414 +425,105 @@ Flake getFlake( return getFlake(state, originalRef, useRegistries, {}, requireLockable); } -static LockFile readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) +std::unique_ptr parseLockFile( + const fetchers::Settings & fetchSettings, + Flake flake, + const nlohmann::json & json, + std::string_view path, + unsigned int versionIfMissing) +{ + auto version = json.is_null() ? versionIfMissing : (unsigned int) json.value("version", 0); + + if (version == 8) + return parseLockFileV8(fetchSettings, std::move(flake), json, path); + else if (version >= 5 && version <= 7) + return parseLockFileV7(fetchSettings, std::move(flake), json, path); + else + throw Error("lock file '%s' has unsupported version %d", path, version); +} + +void warnRegistry( + const InputAttrPath & inputAttrPath, + const FlakeRef & ref, + const FlakeRef & resolvedRef, + const SourcePath & topFlakePath) { - return lockFilePath.pathExists() ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) - : LockFile(); + if (inputAttrPath.size() == 1 && !ref.input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + printInputAttrPath(inputAttrPath), + topFlakePath, + s.str()); + } } -LockedFlake lockFlake( +std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; - auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; if (lockFlags.applyNixConfig) { flake.config.apply(settings); state.store->setOptions(); } + auto flakeRefForTrace = flake.lockedRef.to_string(); + try { if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } - auto oldLockFile = - readLockFile(state.fetchSettings, lockFlags.referenceLockFilePath.value_or(flake.lockFilePath())); - - debug("old lock file: %s", oldLockFile); - - struct OverrideTarget - { - FlakeInput input; - SourcePath sourcePath; - std::optional parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix? - }; - - std::map overrides; - std::set explicitCliOverrides; - std::set overridesUsed; - std::set updatesUsed; - std::map, SourcePath> nodePaths; - - for (auto & i : lockFlags.inputOverrides) { - overrides.emplace( - i.first, - OverrideTarget{ - .input = FlakeInput{.ref = i.second}, - /* Note: any relative overrides - (e.g. `--override-input B/C "path:./foo/bar"`) - are interpreted relative to the top-level - flake. */ - .sourcePath = flake.path, - }); - explicitCliOverrides.insert(i.first); - } - - LockFile newLockFile; - - std::vector parents; - - std::function node, - const InputAttrPath & inputAttrPathPrefix, - std::shared_ptr oldNode, - const InputAttrPath & followsPrefix, - const SourcePath & sourcePath, - bool trustLock)> - computeLocks; - - computeLocks = [&]( - /* The inputs of this node, either from flake.nix or - flake.lock. */ - const FlakeInputs & flakeInputs, - /* The node whose locks are to be updated.*/ - ref node, - /* The path to this node in the lock file graph. */ - const InputAttrPath & inputAttrPathPrefix, - /* The old node, if any, from which locks can be - copied. */ - std::shared_ptr oldNode, - /* The prefix relative to which 'follows' should be - interpreted. When a node is initially locked, it's - relative to the node's flake; when it's already locked, - it's relative to the root of the lock file. */ - const InputAttrPath & followsPrefix, - /* The source path of this node's flake. */ - const SourcePath & sourcePath, - bool trustLock) { - debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix)); - - /* Get the overrides (i.e. attributes of the form - 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - auto addOverrides = - [&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void { - for (auto & [idOverride, inputOverride] : input.overrides) { - auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride); - if (inputOverride.ref || inputOverride.follows) - overrides.emplace( - inputAttrPath, - OverrideTarget{ - .input = inputOverride, - .sourcePath = sourcePath, - .parentInputAttrPath = inputAttrPathPrefix}); - addOverrides(inputOverride, inputAttrPath); - } - }; + auto lockFilePath = lockFlags.referenceLockFilePath.value_or(flake.lockFilePath()); - for (auto & [id, input] : flakeInputs) { - auto inputAttrPath(inputAttrPathPrefix); - inputAttrPath.push_back(id); - addOverrides(input, inputAttrPath); - } + nlohmann::json oldLockFileJson; - /* Check whether this input has overrides for a - non-existent input. */ - for (auto [inputAttrPath, inputOverride] : overrides) { - auto follow = inputAttrPath.inputName(); - auto inputAttrPath2 = inputAttrPath.parent(); - if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow)) - warn( - "input '%s' has an override for a non-existent input '%s'", - printInputAttrPath(inputAttrPathPrefix), - follow); + if (lockFilePath.pathExists()) { + try { + oldLockFileJson = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); } + } - /* Go over the flake inputs, resolve/fetch them if - necessary (i.e. if they're new or the flakeref changed - from what's in the lock file). */ - for (auto & [id, input2] : flakeInputs) { - auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id); - auto inputAttrPath = nonEmptyInputAttrPath.get(); - auto inputAttrPathS = printInputAttrPath(inputAttrPath); - debug("computing input '%s'", inputAttrPathS); - - try { - - /* Do we have an override for this input from one of the - ancestors? */ - auto i = overrides.find(nonEmptyInputAttrPath); - bool hasOverride = i != overrides.end(); - bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); - if (hasOverride) - overridesUsed.insert(nonEmptyInputAttrPath); - auto input = hasOverride ? i->second.input : input2; - - /* Resolve relative 'path:' inputs relative to - the source path of the overrider. */ - auto overriddenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; - - /* Respect the "flakeness" of the input even if we - override it. */ - if (hasOverride) - input.isFlake = input2.isFlake; - - /* Resolve 'follows' later (since it may refer to an input - path we haven't processed yet. */ - if (input.follows) { - InputAttrPath target; - - target.insert(target.end(), input.follows->begin(), input.follows->end()); - - debug("input '%s' follows '%s'", inputAttrPathS, printInputAttrPath(target)); - node->inputs.insert_or_assign(id, target); - continue; - } - - if (!input.ref) - input.ref = - FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}}); - - auto overriddenParentPath = - input.ref->input.isRelative() - ? std::optional( - hasOverride ? i->second.parentInputAttrPath : inputAttrPathPrefix) - : std::nullopt; - - auto resolveRelativePath = [&]() -> std::optional { - if (auto relativePath = input.ref->input.isRelative()) { - return SourcePath{ - overriddenSourcePath.accessor, - CanonPath(relativePath->string(), overriddenSourcePath.path.parent().value())}; - } else - return std::nullopt; - }; - - /* Get the input flake, resolve 'path:./...' - flakerefs relative to the parent flake. */ - auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { - if (auto resolvedPath = resolveRelativePath()) { - return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); - } else { - return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); - } - }; - - /* Do we have an entry in the existing lock file? - And the input is not in updateInputs? */ - std::shared_ptr oldLock; - - updatesUsed.insert(inputAttrPath); - - if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath)) - if (auto oldLock2 = get(oldNode->inputs, id)) - if (auto oldLock3 = std::get_if<0>(&*oldLock2)) - oldLock = *oldLock3; - - if (oldLock && oldLock->originalRef.canonicalize() == input.ref->canonicalize() - && oldLock->parentInputAttrPath == overriddenParentPath && !hasCliOverride) { - debug("keeping existing input '%s'", inputAttrPathS); - - /* Copy the input from the old lock since its flakeref - didn't change and there is no override from a - higher level flake. */ - auto childNode = make_ref( - oldLock->lockedRef, - oldLock->originalRef, - oldLock->isFlake, - oldLock->buildTime, - oldLock->parentInputAttrPath); - - node->inputs.insert_or_assign(id, childNode); - - /* If we have this input in updateInputs, then we - must fetch the flake to update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath); - - auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size() - && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); - - FlakeInputs fakeInputs; - - if (!mustRefetch) { - /* No need to fetch this flake, we can be - lazy. However there may be new overrides on the - inputs of this flake, so we need to check - those. */ - for (auto & i : oldLock->inputs) { - if (auto lockedNode = std::get_if<0>(&i.second)) { - fakeInputs.emplace( - i.first, - FlakeInput{ - .ref = (*lockedNode)->originalRef, - .isFlake = (*lockedNode)->isFlake, - }); - } else if (auto follows = std::get_if<1>(&i.second)) { - if (!trustLock) { - // It is possible that the flake has changed, - // so we must confirm all the follows that are in the lock file are also in the - // flake. - auto overridePath = - NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first); - auto o = overrides.find(overridePath); - // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lock file. - if (o == overrides.end()) { - mustRefetch = true; - // There's no point populating the rest of the fake inputs, - // since we'll refetch the flake anyways. - break; - } - } - auto absoluteFollows(followsPrefix); - absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); - fakeInputs.emplace( - i.first, - FlakeInput{ - .follows = absoluteFollows, - }); - } - } - } - - if (mustRefetch) { - auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); - nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks( - inputFlake.inputs, - childNode, - inputAttrPath, - oldLock, - followsPrefix, - inputFlake.path, - false); - } else { - computeLocks( - fakeInputs, childNode, inputAttrPath, oldLock, followsPrefix, sourcePath, true); - } - - } else { - /* We need to create a new lock file entry. So fetch - this input. */ - debug("creating new input '%s'", inputAttrPathS); - - if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) - && !input.ref->input.isRelative()) - throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); - - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); - auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; - - /* Warn against the use of indirect flakerefs - (but only at top-level since we don't want - to annoy users about flakes that are not - under their control). */ - auto warnRegistry = [&](const FlakeRef & resolvedRef) { - if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { - std::ostringstream s; - printLiteralString(s, resolvedRef.to_string()); - warn( - "Flake input '%1%' uses the flake registry. " - "Using the registry in flake inputs is deprecated in Determinate Nix. " - "To make your flake future-proof, add the following to '%2%':\n" - "\n" - " inputs.%1%.url = %3%;\n" - "\n" - "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", - inputAttrPathS, - flake.path, - s.str()); - } - }; - - if (input.isFlake) { - auto inputFlake = getInputFlake( - *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); - - auto childNode = make_ref( - inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); - - node->inputs.insert_or_assign(id, childNode); - - /* Guard against circular flake imports. */ - for (auto & parent : parents) - if (parent == *input.ref) - throw Error("found circular import of flake '%s'", parent); - parents.push_back(*input.ref); - Finally cleanup([&]() { parents.pop_back(); }); - - /* Recursively process the inputs of this - flake, using its own lock file. */ - nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks( - inputFlake.inputs, - childNode, - inputAttrPath, - readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), - inputAttrPath, - inputFlake.path, - false); - - warnRegistry(inputFlake.resolvedRef); - } - - else { - auto [path, lockedRef] = [&]() -> std::tuple { - // Handle non-flake 'path:./...' inputs. - if (auto resolvedPath = resolveRelativePath()) { - return {*resolvedPath, *input.ref}; - } else { - auto cachedInput = state.inputCache->getAccessor( - state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); - - auto resolvedRef = - FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); - auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - - warnRegistry(resolvedRef); - - return { - state.storePath(state.mountInput( - lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), - lockedRef}; - } - }(); + std::optional oldVersion; + if (!oldLockFileJson.is_null()) { + oldVersion = oldLockFileJson.value("version", 0); + if (*oldVersion < 5 || *oldVersion > 8) + throw Error("lock file '%s' has unsupported version %d", lockFilePath, *oldVersion); + } - auto childNode = - make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); + /* Determine the version of the new lock file: the existing + lock file's version wins, unless `--recreate-lock-file` was + passed (or there is no lock file), in which case the + `lock-file-format` setting is used. */ + unsigned int version = oldVersion && !lockFlags.recreateLockFile ? (*oldVersion == 8 ? 8 : 7) + : (unsigned int) settings.lockFileFormat; - nodePaths.emplace(childNode, path); + if (version != 7 && version != 8) + throw Error("unsupported lock file format version %d; supported versions are 7 and 8", version); - node->inputs.insert_or_assign(id, childNode); - } - } + /* Parse the old lock file. If there is no lock file, get an + empty lock file of the version we're producing, so that the + lock functions below receive the type they expect. */ + auto oldLockedFlake = + parseLockFile(state.fetchSettings, flake, oldLockFileJson, fmt("%s", lockFilePath), version); - } catch (Error & e) { - e.addTrace({}, "while updating the flake input '%s'", inputAttrPathS); - throw; - } - } - }; + debug("old lock file: %s", oldLockedFlake->to_string()); - nodePaths.emplace(newLockFile.root, flake.path.parent()); + auto lockFlakeForVersion = version == 8 ? lockFlakeV8 : lockFlakeV7; - computeLocks( - flake.inputs, - newLockFile.root, - {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), - {}, - flake.path, - false); + auto [lockedFlake, overridesUsed, updatesUsed] = + lockFlakeForVersion(settings, state, lockFlags, std::move(flake), *oldLockedFlake); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -836,25 +532,24 @@ LockedFlake lockFlake( printInputAttrPath(i.first), i.second); - for (auto & i : lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("'%s' does not match any input of this flake", printInputAttrPath(i)); + if (lockFlags.inputUpdates) + for (auto & i : *lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("'%s' does not match any input of this flake", printInputAttrPath(i)); - /* Check 'follows' inputs. */ - newLockFile.check(); - - debug("new lock file: %s", newLockFile); + debug("new lock file: %s", lockedFlake->to_string()); auto sourcePath = topRef.input.getSourcePath(); /* Check whether we need to / can write the new lock file. */ - if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { + auto lockedFlakeJson = lockedFlake->toJSON(); + if (lockedFlakeJson != oldLockedFlake->toJSON() || lockFlags.outputLockFilePath) { - auto diff = LockFile::diff(oldLockFile, newLockFile); + auto diff = diffLockedFlakes(*oldLockedFlake, *lockedFlake, false); if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { - if (auto unlockedInput = newLockFile.isUnlocked(state.fetchSettings)) { + if (auto unlockedInput = lockedFlake->isUnlocked(state.fetchSettings)) { if (lockFlags.failOnUnlocked) throw Error( "Not writing lock file of flake '%s' because it has an unlocked input ('%s'). " @@ -872,7 +567,7 @@ LockedFlake lockFlake( "flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto newLockFileS = fmt("%s\n", newLockFile); + auto newLockFileS = fmt("%s\n", lockedFlakeJson.dump(2)); if (lockFlags.outputLockFilePath) { if (lockFlags.commitLockFile) @@ -918,32 +613,31 @@ LockedFlake lockFlake( /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ - auto prevLockedRef = flake.lockedRef; - flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); + auto prevLockedRef = lockedFlake->flake.lockedRef; + lockedFlake->flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); - if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() - && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) - warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + if (lockFlags.commitLockFile && lockedFlake->flake.lockedRef.input.getRev() + && prevLockedRef.input.getRev() != lockedFlake->flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", lockedFlake->flake.lockedRef.input.getRev()->gitRev()); } } else throw Error( "cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); } else { warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); - flake.forceDirty = true; + lockedFlake->flake.forceDirty = true; } } - return LockedFlake{ - .flake = std::move(flake), .lockFile = std::move(newLockFile), .nodePaths = std::move(nodePaths)}; + return std::move(lockedFlake); } catch (Error & e) { - e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); + e.addTrace({}, "while updating the lock file of flake '%s'", flakeRefForTrace); throw; } } -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); @@ -952,7 +646,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {}, false)); } -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags) { /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */ @@ -981,58 +675,285 @@ static Value * requireInternalFile(EvalState & state, CanonPath path) return v; } -void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) +/** + * An external value wrapping a `LockedFlake`, passed as an argument + * to `call-flake.nix` and consumed by the `listFlakeInputs` and + * `fetchFlakeInput` primops. + */ +class LockedFlakeValue : public ExternalValueBase, public gc_cleanup +{ +public: + const std::shared_ptr lockedFlake; + + LockedFlakeValue(std::shared_ptr lockedFlake) + : lockedFlake(std::move(lockedFlake)) + { + } + + std::string showType() const override + { + return "a locked flake"; + } + + std::string typeOf() const override + { + return "lockedFlake"; + } + +protected: + std::ostream & print(std::ostream & str) const override + { + return str << "«locked flake»"; + } +}; + +static const LockedFlake & requireLockedFlake(EvalState & state, Value & v, const PosIdx pos) +{ + state.forceValue(v, pos); + if (v.type() == nExternal) + if (auto * ext = dynamic_cast(v.external())) + return *ext->lockedFlake; + state.error("expected a locked flake but found %1%", showType(v)).atPos(pos).debugThrow(); +} + +static InputAttrPath getInputAttrPathArg(EvalState & state, Value & v, const PosIdx pos) { - auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); + state.forceList(v, pos, "while evaluating an input attribute path"); + InputAttrPath path; + for (auto elem : v.listView()) + path.push_back( + std::string(state.forceStringNoCtx(*elem, pos, "while evaluating an input attribute path element"))); + return path; +} + +static void prim_listFlakeInputs(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto & lockedFlake = requireLockedFlake(state, *args[0], pos); + auto prefix = getInputAttrPathArg(state, *args[1], pos); + + auto targets = lockedFlake.getInputTargets(state, prefix); + + auto attrs = state.buildBindings(targets.size()); + + for (auto & [id, target] : targets) { + auto & vTarget = attrs.alloc(state.symbols.create(id)); + if (!target) + vTarget.mkNull(); + else { + auto list = state.buildList(target->size()); + for (const auto & [n, elem] : enumerate(*target)) + (list[n] = state.allocValue())->mkString(elem, state.mem); + vTarget.mkList(list); + } + } + + v.mkAttrs(attrs); +} - auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); +static RegisterPrimOp primop_listFlakeInputs({ + .name = "__listFlakeInputs", + .args = {"lockedFlake", "inputAttrPath"}, + .doc = R"( + For the flake input of *lockedFlake* denoted by *inputAttrPath* + (a list of strings, where the empty list denotes the top-level + flake), return an attribute set mapping the names of its inputs + to either null (for a regular input) or the input attribute path + of the target of a "follows" input. + )", + .impl = prim_listFlakeInputs, + .internal = true, +}); + +static void prim_fetchFlakeInput(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + auto & lockedFlake = requireLockedFlake(state, *args[0], pos); + auto path = getInputAttrPathArg(state, *args[1], pos); + + std::optional info; + if (!path.empty()) { + info = lockedFlake.findInput(state, path); + if (!info) + state.error("flake input '%s' does not exist", printInputAttrPath(path)).atPos(pos).debugThrow(); + } - for (auto & [node, sourcePath] : lockedFlake.nodePaths) { - auto override = state.buildBindings(2); + auto attrs = state.buildBindings(4); - auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); + attrs.alloc("flake").mkBool(info ? info->isFlake : true); + attrs.alloc("buildTime").mkBool(info && info->buildTime); - auto lockedNode = node.dynamic_pointer_cast(); + if (info && info->buildTime) { + /* Build-time inputs are not fetched at evaluation time; + return the locked input attributes so that call-flake.nix + can construct a `builtin:fetch-tree` derivation. */ + parseJSON(state, fetchers::attrsToJSON(info->lockedRef.toAttrs()).dump(), attrs.alloc("locked")); + } else { + if (info && !info->lockedRef.input.isRelative()) + state.checkURI(info->lockedRef.input.toURLString()); + auto sourcePath = lockedFlake.getSourcePath(state, path); auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs()); + /* Relative path inputs have the same source tree as their + parent flake, so their `sourceInfo` metadata comes from the + nearest non-relative ancestor. */ + auto info2 = info; + while (info2 && info2->lockedRef.input.isRelative()) { + assert(info2->parentInputAttrPath); + if (info2->parentInputAttrPath->empty()) + /* The parent is the top-level flake. */ + info2.reset(); + else + info2 = lockedFlake.findInput(state, *info2->parentInputAttrPath); + } + emitTreeAttrs( state, storePath, - lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input, - vSourceInfo, + info2 ? info2->lockedRef.input : lockedFlake.flake.lockedRef.input, + attrs.alloc("sourceInfo"), false, - !lockedNode && lockedFlake.flake.forceDirty); + !info2 && lockedFlake.flake.forceDirty); + + attrs.alloc("dir").mkString(CanonPath(subdir).rel(), state.mem); + } + + v.mkAttrs(attrs); +} + +static RegisterPrimOp primop_fetchFlakeInput({ + .name = "__fetchFlakeInput", + .args = {"lockedFlake", "inputAttrPath"}, + .doc = R"( + Fetch the flake input of *lockedFlake* denoted by *inputAttrPath* + (a list of strings, where the empty list denotes the top-level + flake) and return an attribute set describing it: `flake` + (whether it's a flake), `buildTime` (whether it's fetched at + build time), and either `locked` (the locked input attributes, + for build-time inputs, which are not fetched) or `sourceInfo` + (the fetched tree's metadata) and `dir` (the subdirectory of the + flake within `sourceInfo`). + )", + .impl = prim_fetchFlakeInput, + .internal = true, +}); + +void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & vRes) +{ + auto vLockedFlake = state.allocValue(); + vLockedFlake->mkExternal(new LockedFlakeValue(std::move(lockedFlake))); + + Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); - auto key = keyMap.find(node); - assert(key != keyMap.end()); + Value * args[] = { + vLockedFlake, + **get(state.internalPrimOps, "listFlakeInputs"), + **get(state.internalPrimOps, "fetchFlakeInput"), + }; + state.callFunction(*vCallFlake, args, vRes, noPos); +} - override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem); +LockedFlake::~LockedFlake() {} - overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); +std::vector LockedFlake::getInputNames(EvalState & state, const InputAttrPath & prefix) const +{ + std::vector res; + for (auto & [name, target] : getInputTargets(state, prefix)) + res.push_back(name); + return res; +} + +InputAttrPath LockedFlake::resolveFollows(EvalState & state, const InputAttrPath & path) const +{ + std::vector visited; + + InputAttrPath res; + + /* The path elements still to be resolved, in reverse order. */ + InputAttrPath todo(path.rbegin(), path.rend()); + + while (!todo.empty()) { + auto name = todo.back(); + todo.pop_back(); + + auto targets = getInputTargets(state, res); + auto i = targets.find(name); + + if (i == targets.end()) { + /* The input doesn't exist, so there is nothing to + resolve; return the remaining path unchanged and leave + it to the caller to deal with it. */ + res.push_back(std::move(name)); + res.insert(res.end(), todo.rbegin(), todo.rend()); + return res; + } + + if (i->second) { + /* A "follows" input: restart resolution from its target + (which is relative to the top-level flake). */ + auto followsPath(res); + followsPath.push_back(std::move(name)); + if (std::find(visited.begin(), visited.end(), followsPath) != visited.end()) { + std::vector cycle; + std::transform(visited.begin(), visited.end(), std::back_inserter(cycle), printInputAttrPath); + cycle.push_back(printInputAttrPath(followsPath)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(std::move(followsPath)); + todo.insert(todo.end(), i->second->rbegin(), i->second->rend()); + res.clear(); + } else + res.push_back(std::move(name)); } - auto & vOverrides = state.allocValue()->mkAttrs(overrides); + return res; +} - Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); +void LockedFlake::visit(EvalState & state, VisitCallback callback) const +{ + if (!callback({}, InputInfo{.lockedRef = flake.lockedRef})) + return; + + /* Note: this is not a recursive lambda using an explicit object + parameter because that triggers an internal compiler error in + GCC. */ + std::function recurse; + + recurse = [&](const InputAttrPath & prefix) { + for (auto & [id, target] : getInputTargets(state, prefix)) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + if (target) + callback(inputAttrPath, *target); + else if (auto info = findInput(state, inputAttrPath)) { + if (callback(inputAttrPath, *info) && info->isFlake) + recurse(inputAttrPath); + } + } + }; - auto vLocks = state.allocValue(); - vLocks->mkString(lockFileStr, state.mem); + recurse({}); +} - Value * args[] = {vLocks, &vOverrides}; - state.callFunction(*vCallFlake, args, vRes, noPos); +std::string LockedFlake::to_string() const +{ + return toJSON().dump(2); +} + +std::ostream & operator<<(std::ostream & stream, const LockedFlake & lockedFlake) +{ + return stream << lockedFlake.to_string(); } std::optional LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const { - if (lockFile.isUnlocked(fetchSettings)) + if (isUnlocked(fetchSettings)) return std::nullopt; auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; - *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, *this); if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { if (std::get_if(revCount)) { diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 99720c9403a0..93571a5a3112 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -3,10 +3,12 @@ #include "nix/util/types.hh" #include "nix/flake/flakeref.hh" -#include "nix/flake/lockfile.hh" +#include "nix/flake/input-attr-path.hh" #include "nix/expr/value.hh" #include "nix/expr/eval-cache.hh" +#include + namespace nix { class EvalState; @@ -86,7 +88,8 @@ struct Flake FlakeRef resolvedRef; /** - * the specific local store result of invoking the fetcher + * The flakeref returned by the fetcher. Note that this is a misnomer and it might not actually be locked (e.g. a + * dirty Git repo). */ FlakeRef lockedRef; @@ -131,6 +134,13 @@ struct Flake Flake getFlake( EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries, bool requireLockable = true); +Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + fetchers::UseRegistries useRegistries, + const InputAttrPath & lockRootAttrPath, + bool requireLockable); + /** * Fingerprint of a locked flake; used as a cache key. */ @@ -139,18 +149,150 @@ typedef Hash Fingerprint; struct LockedFlake { Flake flake; - LockFile lockFile; + + LockedFlake(Flake && flake) + : flake(std::move(flake)) + { + } + + virtual ~LockedFlake(); + + /** + * For the input denoted by `prefix` (or the top-level flake if + * `prefix` is empty), return a map from the names of its inputs + * to the target of that input: for a regular input, std::nullopt; + * for a "follows" input, the input attribute path (relative to + * the top-level flake) of the immediate target of the + * "follows". Note that the target may itself denote a "follows" + * input. `prefix` must be fully resolved (see + * `resolveFollows()`). Throws an error if `prefix` does not + * denote an existing input. + */ + virtual std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const = 0; + + /** + * Return the names of the inputs of the input denoted by + * `prefix`, or of the top-level flake if `prefix` is empty. + * `prefix` must be fully resolved (see `resolveFollows()`). + */ + std::vector getInputNames(EvalState & state, const InputAttrPath & prefix) const; /** - * Source tree accessors for nodes that have been fetched in - * lockFlake(); in particular, the root node and the overridden - * inputs. + * Resolve any "follows" indirections in `path`, returning an + * input attribute path that denotes the same input but does not + * pass through any "follows" input. Such a *fully resolved* path + * is required by methods like `getInputTargets()`, `findInput()` + * and `getSourcePath()`. Path elements that do not denote + * existing inputs are returned unchanged. */ - std::map, SourcePath> nodePaths; + InputAttrPath resolveFollows(EvalState & state, const InputAttrPath & path) const; + + /** + * Information about a locked input. + */ + struct InputInfo + { + FlakeRef lockedRef; + bool isFlake = true; + bool buildTime = false; + + /** + * For relative path inputs (e.g. 'path:./foo'), the input + * attribute path, relative to the top-level flake, of the + * flake against whose source tree the path is resolved. This + * is the flake whose `flake.nix` *declares* the relative + * path: for an overridden input, that's the flake that + * declares the override, not necessarily the input's parent. + */ + std::optional parentInputAttrPath; + }; + + /** + * Return information about the input denoted by `path`, which + * must be fully resolved (see `resolveFollows()`); an error is + * thrown if it passes through a "follows" input. Returns + * std::nullopt if the input does not exist. + */ + virtual std::optional findInput(EvalState & state, const InputAttrPath & path) const = 0; + + /** + * Return the source path of the input denoted by `inputAttrPath` + * (or of the top-level flake if `inputAttrPath` is empty), + * fetching it if necessary. `inputAttrPath` must be fully + * resolved (see `resolveFollows()`). Note: the returned path is + * backed by `EvalState::rootFS` (i.e. it's a store path, possibly + * a virtual one that has the input's accessor mounted on it if + * lazy trees are enabled), not by the input's original accessor. + */ + virtual SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const = 0; + + /** + * Callback for `visit()`. The second argument is either an + * `InputInfo` for locked inputs, or, for "follows" inputs, the + * input attribute path of the target of the "follows" (relative + * to the top-level flake). The return value denotes whether + * `visit()` should recurse into the inputs of this input. + */ + using VisitCallback = + std::function & input)>; + + /** + * Call `callback` for every transitive input of this flake, + * including the root (which has the empty input attribute + * path). Inputs are visited in depth-first order, parents before + * children. If the callback returns false, we do not recurse into + * the inputs of that input. We never recurse into "follows" + * inputs; their targets are visited under their own paths. + */ + void visit(EvalState & state, VisitCallback callback) const; std::optional getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; + + /** + * Check whether the lock file has any unlocked or non-final + * inputs. If so, return one. + */ + virtual std::optional isUnlocked(const fetchers::Settings & fetchSettings) const = 0; + + /** + * Return the version of this lock file's format (e.g. 7 or 8). + */ + virtual unsigned int version() const = 0; + + /** + * A lock file entry: either the locked flakeref of an input, or, + * for "follows" inputs, the input attribute path of the target of + * the "follows" (relative to the top-level flake). + */ + using LockEntry = std::variant; + + /** + * Return the contents of this lock file as a map from input + * attribute paths to lock entries. If `fetchTransitive` is true, + * inputs that have a lock file of their own may be fetched in + * order to include their transitive locks; otherwise only the + * locks contained in this lock file are returned. + */ + virtual std::map getAllLockEntries(bool fetchTransitive) const = 0; + + virtual nlohmann::json toJSON() const = 0; + + std::string to_string() const; }; +std::ostream & operator<<(std::ostream & stream, const LockedFlake & lockedFlake); + +/** + * Return a human-readable description of the differences between two + * locked flakes (which may use different lock file versions), e.g. + * between the old and new version of a lock file written by + * `lockFlake()`. If `fetchTransitive` is true, transitive lock files + * may be fetched (see `LockedFlake::getAllLockEntries()`). + */ +std::string +diffLockedFlakes(const LockedFlake & oldLockedFlake, const LockedFlake & newLockedFlake, bool fetchTransitive); + struct LockFlags { /** @@ -222,9 +364,10 @@ struct LockFlags /** * Flake inputs to be updated. This means that any existing lock - * for those inputs will be ignored. + * for those inputs will be ignored. `std::nullopt` means that + * *all* inputs will be updated. */ - std::set inputUpdates; + std::optional> inputUpdates = std::set(); /** * Whether to require a locked input. @@ -244,20 +387,93 @@ Flake readFlake( const SourcePath & rootDir, const InputAttrPath & lockRootPath); +/** + * The result of functions like `lockFlakeV7()` that compute a lock + * file for a flake. + */ +struct LockFlakeResult +{ + std::unique_ptr lockedFlake; + + /** + * The elements of `LockFlags::inputOverrides` that matched an + * input of the flake. + */ + std::set overridesUsed; + + /** + * The elements of `LockFlags::inputUpdates` that matched an input + * of the flake. + */ + std::set updatesUsed; +}; + /* * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is * writable. */ -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); -LockedFlake lockFlake( +std::unique_ptr lockFlake( const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake); -LockedFlake +std::unique_ptr lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); -void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); +/** + * Parse a lock file, dispatching on the version of its JSON + * representation. `json` must be null if the lock file doesn't exist, + * in which case an empty lock file of version `versionIfMissing` is + * returned. + */ +std::unique_ptr parseLockFile( + const fetchers::Settings & fetchSettings, + Flake flake, + const nlohmann::json & json, + std::string_view path, + unsigned int versionIfMissing = 7); + +/** + * Parse a lock file in the old graph-based format (versions 5-7). + * `json` must be null if the lock file doesn't exist. + */ +std::unique_ptr parseLockFileV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); + +/** + * Compute a version 7 lock file for `flake`, reusing entries from + * `oldLockFile` (which must have been produced by `parseLockFileV7()`) + * where possible. Note: this does not write the new lock file. + */ +LockFlakeResult lockFlakeV7( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); + +/** + * Parse a lock file in the sparse format (version 8). `json` must be + * null if the lock file doesn't exist. + */ +std::unique_ptr parseLockFileV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path); + +/** + * Compute a version 8 lock file for `flake`, reusing entries from + * `oldLockFile` where possible. If `oldLockFile` was not produced by + * `parseLockFileV8()`, it is ignored. Note: this does not write the + * new lock file. + */ +LockFlakeResult lockFlakeV8( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & oldLockFile); + +void callFlake(EvalState & state, std::shared_ptr lockedFlake, Value & v); } // namespace flake diff --git a/src/libflake/include/nix/flake/input-attr-path.hh b/src/libflake/include/nix/flake/input-attr-path.hh new file mode 100644 index 000000000000..8a686dfbea26 --- /dev/null +++ b/src/libflake/include/nix/flake/input-attr-path.hh @@ -0,0 +1,88 @@ +#pragma once +///@file + +#include "nix/flake/flakeref.hh" + +namespace nix::flake { + +typedef std::vector InputAttrPath; + +/** + * A non-empty input attribute path. + * + * Input attribute paths identify inputs in a flake. An empty path would + * refer to the flake itself rather than an input, which contradicts the + * purpose of operations like override or update. + */ +class NonEmptyInputAttrPath +{ + InputAttrPath path; + + explicit NonEmptyInputAttrPath(InputAttrPath && p) + : path(std::move(p)) + { + assert(!path.empty()); + } + +public: + /** + * Parse and validate a non-empty input attribute path. + * Returns std::nullopt if the path is empty. + */ + static std::optional parse(std::string_view s); + + /** + * Construct from an already-parsed path. + * Returns std::nullopt if the path is empty. + */ + static std::optional make(InputAttrPath path); + + /** + * Append an element to a path, creating a non-empty path. + * This is always safe because adding an element guarantees non-emptiness. + */ + static NonEmptyInputAttrPath append(const InputAttrPath & prefix, const FlakeId & element) + { + InputAttrPath path = prefix; + path.push_back(element); + return NonEmptyInputAttrPath{std::move(path)}; + } + + const InputAttrPath & get() const + { + return path; + } + + operator const InputAttrPath &() const + { + return path; + } + + /** + * Get the final component of the path (the input name). + * For a path like "a/b/c", returns "c". + */ + const FlakeId & inputName() const + { + return path.back(); + } + + /** + * Get the parent path (all components except the last). + * For a path like "a/b/c", returns "a/b". + */ + InputAttrPath parent() const + { + InputAttrPath result = path; + result.pop_back(); + return result; + } + + auto operator<=>(const NonEmptyInputAttrPath & other) const = default; +}; + +InputAttrPath parseInputAttrPath(std::string_view s); + +std::string printInputAttrPath(const InputAttrPath & path); + +} // namespace nix::flake diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh deleted file mode 100644 index 27232b20a669..000000000000 --- a/src/libflake/include/nix/flake/lockfile.hh +++ /dev/null @@ -1,178 +0,0 @@ -#pragma once -///@file - -#include "nix/flake/flakeref.hh" - -#include - -namespace nix { -class Store; -class StorePath; -} // namespace nix - -namespace nix::flake { - -typedef std::vector InputAttrPath; - -/** - * A non-empty input attribute path. - * - * Input attribute paths identify inputs in a flake. An empty path would - * refer to the flake itself rather than an input, which contradicts the - * purpose of operations like override or update. - */ -class NonEmptyInputAttrPath -{ - InputAttrPath path; - - explicit NonEmptyInputAttrPath(InputAttrPath && p) - : path(std::move(p)) - { - assert(!path.empty()); - } - -public: - /** - * Parse and validate a non-empty input attribute path. - * Returns std::nullopt if the path is empty. - */ - static std::optional parse(std::string_view s); - - /** - * Construct from an already-parsed path. - * Returns std::nullopt if the path is empty. - */ - static std::optional make(InputAttrPath path); - - /** - * Append an element to a path, creating a non-empty path. - * This is always safe because adding an element guarantees non-emptiness. - */ - static NonEmptyInputAttrPath append(const InputAttrPath & prefix, const FlakeId & element) - { - InputAttrPath path = prefix; - path.push_back(element); - return NonEmptyInputAttrPath{std::move(path)}; - } - - const InputAttrPath & get() const - { - return path; - } - - operator const InputAttrPath &() const - { - return path; - } - - /** - * Get the final component of the path (the input name). - * For a path like "a/b/c", returns "c". - */ - const FlakeId & inputName() const - { - return path.back(); - } - - /** - * Get the parent path (all components except the last). - * For a path like "a/b/c", returns "a/b". - */ - InputAttrPath parent() const - { - InputAttrPath result = path; - result.pop_back(); - return result; - } - - auto operator<=>(const NonEmptyInputAttrPath & other) const = default; -}; - -struct LockedNode; - -/** - * A node in the lock file. It has outgoing edges to other nodes (its - * inputs). Only the root node has this type; all other nodes have - * type LockedNode. - */ -struct Node : std::enable_shared_from_this -{ - typedef std::variant, InputAttrPath> Edge; - - std::map inputs; - - virtual ~Node() {} -}; - -/** - * A non-root node in the lock file. - */ -struct LockedNode : Node -{ - FlakeRef lockedRef, originalRef; - bool isFlake = true; - bool buildTime = false; - - /* The node relative to which relative source paths - (e.g. 'path:../foo') are interpreted. */ - std::optional parentInputAttrPath; - - LockedNode( - const FlakeRef & lockedRef, - const FlakeRef & originalRef, - bool isFlake = true, - bool buildTime = false, - std::optional parentInputAttrPath = {}) - : lockedRef(std::move(lockedRef)) - , originalRef(std::move(originalRef)) - , isFlake(isFlake) - , buildTime(buildTime) - , parentInputAttrPath(std::move(parentInputAttrPath)) - { - } - - LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json); - - StorePath computeStorePath(Store & store) const; -}; - -struct LockFile -{ - ref root = make_ref(); - - LockFile() {}; - LockFile(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path); - - typedef std::map, std::string> KeyMap; - - std::pair toJSON() const; - - std::pair to_string() const; - - /** - * Check whether this lock file has any unlocked or non-final - * inputs. If so, return one. - */ - std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; - - bool operator==(const LockFile & other) const; - - std::shared_ptr findInput(const InputAttrPath & path); - - std::map getAllInputs() const; - - static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - - /** - * Check that every 'follows' input target exists. - */ - void check(); -}; - -std::ostream & operator<<(std::ostream & stream, const LockFile & lockFile); - -InputAttrPath parseInputAttrPath(std::string_view s); - -std::string printInputAttrPath(const InputAttrPath & path); - -} // namespace nix::flake diff --git a/src/libflake/include/nix/flake/meson.build b/src/libflake/include/nix/flake/meson.build index fbe54f41208b..dbe66ef229b3 100644 --- a/src/libflake/include/nix/flake/meson.build +++ b/src/libflake/include/nix/flake/meson.build @@ -5,7 +5,7 @@ include_dirs = [ include_directories('../..') ] headers = files( 'flake.hh', 'flakeref.hh', - 'lockfile.hh', + 'input-attr-path.hh', 'provenance.hh', 'settings.hh', 'url-name.hh', diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 05b36f5b779c..d5e5566cfa6d 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -41,6 +41,19 @@ struct Settings : public Config )", {"commit-lockfile-summary"}, true}; + + Setting lockFileFormat{ + this, + 7, + "lock-file-format", + R"( + The lock file format version to use when creating a new lock + file (7 or 8). An existing lock file keeps its version + unless `--recreate-lock-file` is passed. Note: version 8 + requires the `lock-file-v8` experimental feature. + )", + {}, + true}; }; } // namespace nix::flake diff --git a/src/libflake/input-attr-path.cc b/src/libflake/input-attr-path.cc new file mode 100644 index 000000000000..46ee6e720454 --- /dev/null +++ b/src/libflake/input-attr-path.cc @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include + +#include "nix/flake/input-attr-path.hh" +#include "nix/flake/flakeref.hh" +#include "nix/util/error.hh" +#include "nix/util/strings.hh" + +namespace nix::flake { + +InputAttrPath parseInputAttrPath(std::string_view s) +{ + InputAttrPath path; + + for (auto & elem : tokenizeString>(s, "/")) { + if (!std::regex_match(elem, flakeIdRegex)) + throw UsageError("invalid flake input attribute path element '%s'", elem); + path.push_back(elem); + } + + return path; +} + +std::string printInputAttrPath(const InputAttrPath & path) +{ + return concatStringsSep("/", path); +} + +std::optional NonEmptyInputAttrPath::parse(std::string_view s) +{ + auto path = parseInputAttrPath(s); + return make(std::move(path)); +} + +std::optional NonEmptyInputAttrPath::make(InputAttrPath path) +{ + if (path.empty()) + return std::nullopt; + return NonEmptyInputAttrPath{std::move(path)}; +} + +} // namespace nix::flake diff --git a/src/libflake/lockfile-v7.cc b/src/libflake/lockfile-v7.cc new file mode 100644 index 000000000000..d10aa093d461 --- /dev/null +++ b/src/libflake/lockfile-v7.cc @@ -0,0 +1,912 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nix/fetchers/fetch-settings.hh" +#include "nix/flake/flake.hh" +#include "flake-impl.hh" +#include "nix/util/sync.hh" +#include "nix/flake/settings.hh" +#include "nix/expr/eval.hh" +#include "nix/expr/nixexpr.hh" +#include "nix/store/store-api.hh" +#include "nix/fetchers/input-cache.hh" +#include "nix/util/finally.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/strings.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/flake/flakeref.hh" +#include "nix/store/path.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/error.hh" +#include "nix/util/fmt.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/logging.hh" +#include "nix/util/ref.hh" +#include "nix/util/types.hh" +#include "nix/util/util.hh" + +namespace nix { +class Store; +} // namespace nix + +namespace nix::flake { + +static FlakeRef +getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = fetchers::jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : fetchers::jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(fetchSettings, attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + +struct LockedNode; + +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ +struct Node : std::enable_shared_from_this +{ + typedef std::variant, InputAttrPath> Edge; + + std::map inputs; + + virtual ~Node() {} +}; + +/** + * A non-root node in the lock file. + */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + bool buildTime = false; + std::optional parentInputAttrPath; + + /** + * The source path of this node, if it has been fetched. Set by + * `lockFlakeV7()` for nodes fetched during locking, and by + * `LockedFlakeV7::getSourcePath()` for nodes fetched on + * demand. + */ + mutable Sync> sourcePath; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true, + bool buildTime = false, + std::optional parentInputAttrPath = {}) + : lockedRef(std::move(lockedRef)) + , originalRef(std::move(originalRef)) + , isFlake(isFlake) + , buildTime(buildTime) + , parentInputAttrPath(std::move(parentInputAttrPath)) + { + } + + LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) + , parentInputAttrPath( + json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) + { + if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { + if (lockedRef.input.getNarHash()) + warn( + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", + lockedRef.to_string()); + else + throw Error( + "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); + } + + // For backward compatibility, lock file entries are implicitly final. + assert(!lockedRef.input.attrs.contains("__final")); + lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); + } +}; + +/** + * The old graph-based lock file format (versions 5-7). + */ +struct LockFileV7 +{ + ref root = make_ref(); + + /** + * The version of the lock file this was parsed from (5-7), or 7 + * for new lock files. + */ + unsigned int version = 7; + + LockFileV7() {}; + + LockFileV7(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) + { + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + this->version = version; + + std::string rootKey = json["root"]; + std::map> nodeMap{{rootKey, root}}; + + [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { + if (jsonNode.find("inputs") == jsonNode.end()) + return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { // FIXME: remove, obsolete + InputAttrPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto & nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = make_ref(fetchSettings, *jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, *jsonNode2); + } + if (auto child = k->second.dynamic_pointer_cast()) + node.inputs.insert_or_assign(i.key(), ref(child)); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); + } + } + }(*root, json["nodes"][rootKey]); + + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. + } + + nlohmann::json toJSON() const + { + nlohmann::json nodes; + std::map, std::string> nodeKeys; + boost::unordered_flat_set keys; + + auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2;; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } + } + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = node.dynamic_pointer_cast()) { + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); + assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); + if (!lockedNode->isFlake) + n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; + if (lockedNode->parentInputAttrPath) + n["parent"] = *lockedNode->parentInputAttrPath; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return json; + } + + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const + { + std::set> nodes; + + [&](this const auto & visit, ref node) { + if (!nodes.insert(node).second) + return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }(root); + + /* Return whether the input is either locked, or, if + `allow-dirty-locks` is enabled, it has a NAR hash. In the + latter case, we can verify the input but we may not be able to + fetch it from anywhere. */ + auto isConsideredLocked = [&](const fetchers::Input & input) { + return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); + }; + + for (auto & i : nodes) { + if (i == ref(root)) + continue; + auto node = i.dynamic_pointer_cast(); + if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) + && !node->lockedRef.input.isRelative()) + return node->lockedRef; + } + + return {}; + } + + bool operator==(const LockFileV7 & other) const + { + // FIXME: slow + return toJSON() == other.toJSON(); + } + + /** + * Return the node denoted by `path`, which must be fully + * resolved: an error is thrown if it passes through a 'follows' + * edge. Returns null if the input doesn't exist. + */ + std::shared_ptr findInput(const InputAttrPath & path) const + { + std::shared_ptr pos = root.get_ptr(); + + for (const auto & [n, elem] : enumerate(path)) { + auto i = get(pos->inputs, elem); + if (!i) + return nullptr; + if (auto node = std::get_if<0>(&*i)) + pos = node->get_ptr(); + else + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath({path.begin(), path.begin() + n + 1})); + } + + return pos; + } + + std::map getAllInputs() const + { + std::set> done; + std::map res; + + [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { + if (!done.insert(node).second) + return; + + for (auto & [id, input] : node->inputs) { + auto inputAttrPath(prefix); + inputAttrPath.push_back(id); + res.emplace(inputAttrPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputAttrPath, *child); + } + }({}, root); + + return res; + } +}; + +struct LockedFlakeV7 : LockedFlake +{ + /** + * The lock file in the old graph-based format (versions 5-7). + */ + LockFileV7 lockFile; + + LockedFlakeV7(Flake && flake, LockFileV7 && lockFile) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + { + } + + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV7() : LockFileV7(fetchSettings, json, path)) + { + } + + std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const override + { + auto node = lockFile.findInput(prefix); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); + + std::map> res; + + for (auto & [id, input] : node->inputs) { + if (std::get_if<0>(&input)) + res.emplace(id, std::nullopt); + else + res.emplace(id, std::get<1>(input)); + } + + return res; + } + + std::optional findInput(EvalState & state, const InputAttrPath & path) const override + { + if (auto node = std::dynamic_pointer_cast(lockFile.findInput(path))) + return InputInfo{ + .lockedRef = node->lockedRef, + .isFlake = node->isFlake, + .buildTime = node->buildTime, + .parentInputAttrPath = node->parentInputAttrPath, + }; + return std::nullopt; + } + + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override + { + /* The root node. */ + if (inputAttrPath.empty()) + return flake.path.parent(); + + auto node = lockFile.findInput(inputAttrPath); + if (!node) + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + + auto lockedNode = std::dynamic_pointer_cast(node); + assert(lockedNode); + + { + auto sourcePath(lockedNode->sourcePath.lock()); + if (*sourcePath) + return **sourcePath; + } + + /* Note: we fetch without holding the `sourcePath` lock, so + concurrent calls don't get serialized. Racing fetches of the + same node are harmless since they produce the same path. */ + auto path = [&]() -> SourcePath { + if (auto relativePath = lockedNode->lockedRef.input.isRelative()) { + /* Resolve relative path inputs against the source path of + their parent flake. */ + auto parentPath = getSourcePath(state, lockedNode->parentInputAttrPath.value()); + return {parentPath.accessor, CanonPath(relativePath->string(), parentPath.path)}; + } else { + /* Note: `lockedRef` is a copy since `mountInput()` may + modify the input (e.g. adding a `narHash` attribute). */ + auto lockedRef = lockedNode->lockedRef; + auto accessor = + state.inputCache + ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) + .accessor; + return state.storePath( + state.mountInput(lockedRef.input, lockedNode->lockedRef.input, accessor, true, true)) + / CanonPath(lockedRef.subdir); + } + }(); + + *lockedNode->sourcePath.lock() = path; + + return path; + } + + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override + { + return lockFile.isUnlocked(fetchSettings); + } + + unsigned int version() const override + { + return lockFile.version; + } + + std::map getAllLockEntries(bool fetchTransitive) const override + { + /* Note: `fetchTransitive` is irrelevant, since a version 7 + lock file already contains all transitive locks. */ + std::map res; + + for (auto & [path, edge] : lockFile.getAllInputs()) { + if (auto node = std::get_if<0>(&edge)) + res.emplace(path, (*node)->lockedRef); + else + res.emplace(path, std::get<1>(edge)); + } + + return res; + } + + nlohmann::json toJSON() const override + { + return lockFile.toJSON(); + } +}; + +static LockFileV7 readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) +{ + if (!lockFilePath.pathExists()) + return LockFileV7(); + + auto json = [&] { + try { + return nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + }(); + + return LockFileV7(fetchSettings, json, fmt("%s", lockFilePath)); +} + +std::unique_ptr parseLockFileV7( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) +{ + return std::make_unique(fetchSettings, std::move(flake), json, path); +} + +LockFlakeResult lockFlakeV7( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & _oldLockFile) +{ + /* If the old lock file is not a version 7 lock file (e.g. when + migrating from version 8), ignore it. */ + auto oldLockedFlake = dynamic_cast(&_oldLockFile); + LockFileV7 emptyLockFile; + auto & oldLockFile = oldLockedFlake ? oldLockedFlake->lockFile : emptyLockFile; + + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; + + struct OverrideTarget + { + FlakeInput input; + SourcePath sourcePath; + std::optional parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix? + }; + + std::map overrides; + std::set explicitCliOverrides; + std::set overridesUsed; + std::set updatesUsed; + + for (auto & i : lockFlags.inputOverrides) { + overrides.emplace( + i.first, + OverrideTarget{ + .input = FlakeInput{.ref = i.second}, + /* Note: any relative overrides + (e.g. `--override-input B/C "path:./foo/bar"`) + are interpreted relative to the top-level + flake. */ + .sourcePath = flake.path, + }); + explicitCliOverrides.insert(i.first); + } + + LockFileV7 newLockFile; + + std::vector parents; + + std::function node, + const InputAttrPath & inputAttrPathPrefix, + std::shared_ptr oldNode, + const InputAttrPath & followsPrefix, + const SourcePath & sourcePath, + bool trustLock)> + computeLocks; + + computeLocks = [&]( + /* The inputs of this node, either from flake.nix or + flake.lock. */ + const FlakeInputs & flakeInputs, + /* The node whose locks are to be updated.*/ + ref node, + /* The path to this node in the lock file graph. */ + const InputAttrPath & inputAttrPathPrefix, + /* The old node, if any, from which locks can be + copied. */ + std::shared_ptr oldNode, + /* The prefix relative to which 'follows' should be + interpreted. When a node is initially locked, it's + relative to the node's flake; when it's already locked, + it's relative to the root of the lock file. */ + const InputAttrPath & followsPrefix, + /* The source path of this node's flake. */ + const SourcePath & sourcePath, + bool trustLock) { + debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + auto addOverrides = + [&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride); + if (inputOverride.ref || inputOverride.follows) + overrides.emplace( + inputAttrPath, + OverrideTarget{ + .input = inputOverride, + .sourcePath = sourcePath, + .parentInputAttrPath = inputAttrPathPrefix}); + addOverrides(inputOverride, inputAttrPath); + } + }; + + for (auto & [id, input] : flakeInputs) { + auto inputAttrPath(inputAttrPathPrefix); + inputAttrPath.push_back(id); + addOverrides(input, inputAttrPath); + } + + /* Check whether this input has overrides for a + non-existent input. */ + for (auto [inputAttrPath, inputOverride] : overrides) { + auto follow = inputAttrPath.inputName(); + auto inputAttrPath2 = inputAttrPath.parent(); + if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow)) + warn( + "input '%s' has an override for a non-existent input '%s'", + printInputAttrPath(inputAttrPathPrefix), + follow); + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id); + auto inputAttrPath = nonEmptyInputAttrPath.get(); + auto inputAttrPathS = printInputAttrPath(inputAttrPath); + debug("computing input '%s'", inputAttrPathS); + + try { + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(nonEmptyInputAttrPath); + bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); + if (hasOverride) + overridesUsed.insert(nonEmptyInputAttrPath); + auto input = hasOverride ? i->second.input : input2; + + /* Resolve relative 'path:' inputs relative to + the source path of the overrider. */ + auto overriddenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; + + /* Respect the "flakeness" of the input even if we + override it. */ + if (hasOverride) + input.isFlake = input2.isFlake; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputAttrPath target; + + target.insert(target.end(), input.follows->begin(), input.follows->end()); + + debug("input '%s' follows '%s'", inputAttrPathS, printInputAttrPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } + + if (!input.ref) + input.ref = + FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}}); + + auto overriddenParentPath = input.ref->input.isRelative() + ? std::optional( + hasOverride ? i->second.parentInputAttrPath : inputAttrPathPrefix) + : std::nullopt; + + auto resolveRelativePath = [&]() -> std::optional { + if (auto relativePath = input.ref->input.isRelative()) { + return SourcePath{ + overriddenSourcePath.accessor, + CanonPath(relativePath->string(), overriddenSourcePath.path.parent().value())}; + } else + return std::nullopt; + }; + + /* Get the input flake, resolve 'path:./...' + flakerefs relative to the parent flake. */ + auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { + if (auto resolvedPath = resolveRelativePath()) { + return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); + } else { + return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); + } + }; + + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ + std::shared_ptr oldLock; + + updatesUsed.insert(inputAttrPath); + + if (oldNode && !(lockFlags.inputUpdates && lockFlags.inputUpdates->count(nonEmptyInputAttrPath))) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; + + if (oldLock && oldLock->originalRef.canonicalize() == input.ref->canonicalize() + && oldLock->parentInputAttrPath == overriddenParentPath && !hasCliOverride) { + debug("keeping existing input '%s'", inputAttrPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = make_ref( + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->buildTime, + oldLock->parentInputAttrPath); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ + auto mustRefetch = false; + + if (lockFlags.inputUpdates) { + auto lb = lockFlags.inputUpdates->lower_bound(nonEmptyInputAttrPath); + + mustRefetch = lb != lockFlags.inputUpdates->end() && lb->get().size() > inputAttrPath.size() + && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); + } + + FlakeInputs fakeInputs; + + if (!mustRefetch) { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace( + i.first, + FlakeInput{ + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + if (!trustLock) { + // It is possible that the flake has changed, + // so we must confirm all the follows that are in the lock file are also in the + // flake. + auto overridePath = NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first); + auto o = overrides.find(overridePath); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lock file. + if (o == overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } + } + auto absoluteFollows(followsPrefix); + absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); + fakeInputs.emplace( + i.first, + FlakeInput{ + .follows = absoluteFollows, + }); + } + } + } + + if (mustRefetch) { + auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); + *childNode->sourcePath.lock() = inputFlake.path.parent(); + computeLocks( + inputFlake.inputs, + childNode, + inputAttrPath, + oldLock, + followsPrefix, + inputFlake.path, + false); + } else { + computeLocks(fakeInputs, childNode, inputAttrPath, oldLock, followsPrefix, sourcePath, true); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputAttrPathS); + + if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) + && !input.ref->input.isRelative()) + throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); + auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; + + if (input.isFlake) { + auto inputFlake = getInputFlake( + *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); + + auto childNode = make_ref( + inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake, using its own lock file. */ + *childNode->sourcePath.lock() = inputFlake.path.parent(); + computeLocks( + inputFlake.inputs, + childNode, + inputAttrPath, + readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + inputAttrPath, + inputFlake.path, + false); + + warnRegistry(inputAttrPath, *input.ref, inputFlake.resolvedRef, flake.path); + } + + else { + auto [path, lockedRef] = [&]() -> std::tuple { + // Handle non-flake 'path:./...' inputs. + if (auto resolvedPath = resolveRelativePath()) { + return {*resolvedPath, *input.ref}; + } else { + auto cachedInput = state.inputCache->getAccessor( + state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); + + warnRegistry(inputAttrPath, *input.ref, resolvedRef, flake.path); + + return { + state.storePath(state.mountInput( + lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), + lockedRef}; + } + }(); + + auto childNode = + make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); + + *childNode->sourcePath.lock() = path; + + node->inputs.insert_or_assign(id, childNode); + } + } + + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", inputAttrPathS); + throw; + } + } + }; + + computeLocks( + flake.inputs, + newLockFile.root, + {}, + lockFlags.recreateLockFile || !lockFlags.inputUpdates ? nullptr : oldLockFile.root.get_ptr(), + {}, + flake.path, + false); + + auto lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)); + + /* Check that the target of every 'follows' input exists. */ + for (auto & [inputAttrPath, input] : lockedFlake->lockFile.getAllInputs()) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !lockedFlake->lockFile.findInput(lockedFlake->resolveFollows(state, *follows))) + throw Error( + "input '%s' follows a non-existent input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(*follows)); + } + } + + return { + .lockedFlake = std::move(lockedFlake), + .overridesUsed = std::move(overridesUsed), + .updatesUsed = std::move(updatesUsed), + }; +} + +} // namespace nix::flake diff --git a/src/libflake/lockfile-v8.cc b/src/libflake/lockfile-v8.cc new file mode 100644 index 000000000000..d0b6abb988e2 --- /dev/null +++ b/src/libflake/lockfile-v8.cc @@ -0,0 +1,900 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nix/fetchers/fetch-settings.hh" +#include "nix/flake/flake.hh" +#include "flake-impl.hh" +#include "nix/flake/settings.hh" +#include "nix/util/sync.hh" +#include "nix/expr/eval.hh" +#include "nix/store/store-api.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/input-cache.hh" +#include "nix/flake/flakeref.hh" +#include "nix/util/ansicolor.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/configuration.hh" +#include "nix/util/error.hh" +#include "nix/util/experimental-features.hh" +#include "nix/util/finally.hh" +#include "nix/util/fmt.hh" +#include "nix/util/logging.hh" +#include "nix/util/types.hh" +#include "nix/util/util.hh" + +namespace nix::flake { + +static FlakeRef getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr) +{ + auto i = json.find(attr); + if (i == json.end()) + throw Error("attribute '%s' missing in lock file", attr); + return FlakeRef::fromAttrs(fetchSettings, fetchers::jsonToAttrs(*i)); +} + +/** + * The sparse lock file format (version 8). Unlike the graph-based + * versions 5-7, it only stores the immediate inputs of the flake, + * plus any overrides of transitive inputs; the rest of the dependency + * graph is resolved at evaluation time from the inputs' own lock + * files. + */ +struct LockFileV8 +{ + struct Lock + { + FlakeRef originalRef, lockedRef; + + /** + * The locks for the transitive inputs of this input. Only + * present if the input does not have a lock file of its + * own. The keys are relative to this input. + */ + std::unique_ptr locks; + + /** + * The source path of this input, if it has been fetched. + */ + mutable Sync> sourcePath; + + Lock(FlakeRef originalRef, FlakeRef lockedRef) + : originalRef(std::move(originalRef)) + , lockedRef(std::move(lockedRef)) + { + } + + Lock(const fetchers::Settings & fetchSettings, const nlohmann::json & json) + : originalRef(getFlakeRef(fetchSettings, json, "original")) + , lockedRef(getFlakeRef(fetchSettings, json, "locked")) + { + if (!lockedRef.input.isLocked(fetchSettings)) { + if (lockedRef.input.getNarHash()) + warn( + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", + lockedRef.to_string()); + else + throw Error( + "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); + } + + // For backward compatibility, lock file entries are implicitly final. + assert(!lockedRef.input.attrs.contains("__final")); + lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); + + if (auto locks = json.find("locks"); locks != json.end()) + this->locks = std::make_unique(fetchSettings, *locks); + } + + /** + * Return a deep copy of this lock, without the `sourcePath` + * cache. (An explicit method since `Sync` is not copyable.) + */ + Lock clone() const + { + Lock lock(originalRef, lockedRef); + if (locks) + lock.locks = std::make_unique(locks->clone()); + return lock; + } + + nlohmann::json toJSON() const + { + nlohmann::json n; + n["original"] = fetchers::attrsToJSON(originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); + assert(lockedRef.input.isFinal()); + if (locks) + n["locks"] = locks->toLocksJSON(); + return n; + } + }; + + /** + * The locks, keyed by slash-separated input attribute paths. A + * single-element key denotes an immediate input of the flake; + * longer keys denote overrides of transitive inputs. + */ + std::map locks; + + LockFileV8() = default; + + /** + * Construct from a `locks` attribute of a lock file. + */ + LockFileV8(const fetchers::Settings & fetchSettings, const nlohmann::json & locksJson) + { + for (auto & i : locksJson.items()) { + auto path = NonEmptyInputAttrPath::parse(i.key()); + if (!path) + throw Error("lock file contains an empty input attribute path"); + locks.emplace(std::move(*path), Lock(fetchSettings, i.value())); + } + } + + /** + * Construct from the JSON contents of a lock file. + */ + LockFileV8(const fetchers::Settings & fetchSettings, const nlohmann::json & json, std::string_view path) + { + auto version = json.value("version", 0); + if (version != 8) + throw Error("lock file '%s' has unsupported version %d", path, version); + + if (auto locks = json.find("locks"); locks != json.end()) + *this = LockFileV8(fetchSettings, *locks); + } + + LockFileV8 clone() const + { + LockFileV8 res; + for (auto & [path, lock] : locks) + res.locks.emplace(path, lock.clone()); + return res; + } + + nlohmann::json toLocksJSON() const + { + auto res = nlohmann::json::object(); + for (auto & [path, lock] : locks) + res[printInputAttrPath(path)] = lock.toJSON(); + return res; + } + + nlohmann::json toJSON() const + { + nlohmann::json json; + json["version"] = 8; + json["locks"] = toLocksJSON(); + return json; + } + + /** + * Check whether this lock file has any unlocked or non-final + * inputs. If so, return one. + */ + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const + { + /* Return whether the input is either locked, or, if + `allow-dirty-locks` is enabled, it has a NAR hash. In the + latter case, we can verify the input but we may not be able to + fetch it from anywhere. */ + auto isConsideredLocked = [&](const fetchers::Input & input) { + return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); + }; + + for (auto & [path, lock] : locks) { + if (!isConsideredLocked(lock.lockedRef.input) || !lock.lockedRef.input.isFinal()) + return lock.lockedRef; + if (lock.locks) + if (auto ref = lock.locks->isUnlocked(fetchSettings)) + return ref; + } + + return std::nullopt; + } + + /** + * Flatten this lock file into a map from absolute input attribute + * paths to lock entries. Entries in the nested `locks` of an + * entry appear under the path of that entry. If a top-level + * (override) entry denotes the same path as a nested entry, the + * former takes precedence, matching the precedence of overrides + * at evaluation time. + */ + void + getAllLockEntries(std::map & res, const InputAttrPath & prefix = {}) const + { + for (auto & [path, lock] : locks) { + InputAttrPath absPath(prefix); + absPath.insert(absPath.end(), path.get().begin(), path.get().end()); + if (lock.locks) + lock.locks->getAllLockEntries(res, absPath); + /* Note: this takes precedence over any nested entry for + the same path, since the entry for the containing input + sorts before the override path and thus has been + recursed into already. */ + res.insert_or_assign(std::move(absPath), lock.lockedRef); + } + } + + /** + * Return the lock for the exact input attribute path `path`, if + * any. + */ + const Lock * findLock(const InputAttrPath & path) const + { + if (auto p = NonEmptyInputAttrPath::make(path)) { + auto i = locks.find(*p); + if (i != locks.end()) + return &i->second; + } + return nullptr; + } +}; + +/** + * Read and parse the lock file of `flake` (which may use any lock + * file version). The lock file must exist. + */ +static std::unique_ptr readLockFile(EvalState & state, Flake flake) +{ + auto lockFilePath = flake.lockFilePath(); + + nlohmann::json json; + + try { + json = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + + return parseLockFile(state.fetchSettings, std::move(flake), json, fmt("%s", lockFilePath)); +} + +struct LockedFlakeV8 : LockedFlake +{ + /** + * The lock file in the sparse format (version 8). + */ + LockFileV8 lockFile; + + LockedFlakeV8(Flake && flake, LockFileV8 && lockFile) + : LockedFlake(std::move(flake)) + , lockFile(std::move(lockFile)) + { + } + + /** + * Construct from the JSON contents of a lock file (which must be + * null if the lock file doesn't exist). + */ + LockedFlakeV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) + : LockedFlake(std::move(flake)) + , lockFile(json.is_null() ? LockFileV8() : LockFileV8(fetchSettings, json, path)) + { + } + + /** + * Cache of the child `LockedFlake`s to which we delegate the + * resolution of transitive inputs, keyed by the input attribute + * path of their lock file entry (or of a relative path input). + */ + mutable Sync>> children; + + /** + * Look up the `FlakeInput` declared by this flake's `flake.nix` + * for `path`: for a single-element path, the input declaration; + * for longer paths, an override + * (e.g. `inputs.foo.inputs.bar.url = ...`). + */ + const FlakeInput * lookupDeclaration(const InputAttrPath & path) const + { + const FlakeInputs * inputs = &flake.inputs; + const FlakeInput * input = nullptr; + for (auto & elem : path) { + input = get(*inputs, elem); + if (!input) + return nullptr; + inputs = &input->overrides; + } + return input; + } + + /** + * Return the input attribute path of the entry that governs the + * resolution below `path`: the longest prefix of `path` (a proper + * prefix if `proper` is set) that has a lock file entry, or the + * first element of `path` if it denotes a relative path input + * (which is not stored in the lock file). Throws an error if a + * prefix of `path` denotes an input that this flake declares as a + * "follows", since `path` is then not fully resolved (see + * `LockedFlake::resolveFollows()`); note that unlike version 7, + * which stores "follows" in the lock file, we have to check the + * `flake.nix` declarations for this. + */ + std::optional findGoverningKey(const InputAttrPath & path, bool proper) const + { + std::optional res; + + InputAttrPath prefix; + for (auto & elem : path) { + prefix.push_back(elem); + if (auto decl = lookupDeclaration(prefix); decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath(prefix)); + if ((!proper || prefix.size() < path.size()) && lockFile.findLock(prefix)) + res = prefix; + } + + if (res) + return res; + + auto decl = get(flake.inputs, path.front()); + if (decl && decl->ref && decl->ref->input.isRelative()) + return InputAttrPath{path.front()}; + + return std::nullopt; + } + + /** + * Return the child `LockedFlake` that governs the resolution of + * the transitive inputs of the input denoted by `key` (the path + * of an entry in our lock file, or of a relative path input). + */ + std::shared_ptr getChild(EvalState & state, const InputAttrPath & key) const + { + { + auto children_(children.lock()); + if (auto i = get(*children_, key)) + return *i; + } + + /* Note: we construct the child without holding the `children` + lock, so concurrent calls don't get serialized. Racing + constructions of the same child are harmless since they + produce equivalent children. */ + std::shared_ptr child; + + if (auto lock = lockFile.findLock(key)) { + auto inputFlake = getFlake(state, lock->lockedRef, fetchers::UseRegistries::No, {}, false); + + { + auto sourcePath(lock->sourcePath.lock()); + if (!*sourcePath) + *sourcePath = inputFlake.path.parent(); + } + + if (lock->locks) + /* The input doesn't have a lock file of its own; its + transitive inputs are locked in the nested `locks` + of its entry. */ + child = std::make_shared(std::move(inputFlake), lock->locks->clone()); + else if (inputFlake.lockFilePath().pathExists()) + child = readLockFile(state, std::move(inputFlake)); + else + throw Error( + "flake input '%s' does not have a lock file, and its transitive inputs are not locked in the lock file of flake '%s'", + printInputAttrPath(key), + flake.lockedRef); + } else { + /* A relative path input (e.g. 'path:./foo'), resolved + against our own source tree. It is required to have a + lock file of its own. */ + auto decl = lookupDeclaration(key); + assert(decl && decl->ref); + auto relativePath = decl->ref->input.isRelative(); + assert(relativePath); + SourcePath resolvedPath{ + flake.path.accessor, CanonPath(relativePath->string(), flake.path.path.parent().value())}; + auto inputFlake = readFlake(state, *decl->ref, *decl->ref, *decl->ref, resolvedPath, {}); + if (!inputFlake.lockFilePath().pathExists()) + throw Error( + "relative path input '%s' does not have a lock file; run 'nix flake lock %s' to create it", + printInputAttrPath(key), + resolvedPath); + child = readLockFile(state, std::move(inputFlake)); + } + + auto children_(children.lock()); + return children_->emplace(key, std::move(child)).first->second; + } + + std::map> + getInputTargets(EvalState & state, const InputAttrPath & prefix) const override + { + std::map> res; + + if (prefix.empty()) { + for (auto & [name, input] : flake.inputs) + res.emplace(name, input.follows); + return res; + } + + /* Delegate to the child that governs `prefix`. */ + auto key = findGoverningKey(prefix, false); + if (!key) + throw Error("flake input '%s' does not exist", printInputAttrPath(prefix)); + + auto child = getChild(state, *key); + InputAttrPath rest(prefix.begin() + key->size(), prefix.end()); + + for (auto & [name, target] : child->getInputTargets(state, rest)) { + if (target) { + /* Translate the target from child-relative to our + root. */ + InputAttrPath absTarget(*key); + absTarget.insert(absTarget.end(), target->begin(), target->end()); + res.emplace(name, std::move(absTarget)); + } else + res.emplace(name, std::nullopt); + } + + /* Apply the 'follows' overrides declared by this flake for + the inputs of `prefix`; they take precedence over whatever + the child declares. */ + for (auto & [name, target] : res) { + InputAttrPath path(prefix); + path.push_back(name); + auto decl = lookupDeclaration(path); + if (decl && decl->follows) + target = *decl->follows; + } + + return res; + } + + std::optional findInput(EvalState & state, const InputAttrPath & path) const override + { + if (path.empty()) + return std::nullopt; + + auto lock = lockFile.findLock(path); + + if (path.size() == 1) { + auto decl = get(flake.inputs, path.front()); + + if (decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(path), + printInputAttrPath(path)); + + if (!decl) { + if (!lock) + return std::nullopt; + /* A lock file entry without a corresponding + declaration in flake.nix. */ + return InputInfo{.lockedRef = lock->lockedRef}; + } + + if (decl->ref && decl->ref->input.isRelative()) + /* Relative path inputs are not stored in the lock + file. */ + return InputInfo{ + .lockedRef = *decl->ref, + .isFlake = decl->isFlake, + .buildTime = decl->buildTime, + .parentInputAttrPath = InputAttrPath{}}; + + if (!lock) + throw Error( + "lock file of flake '%s' does not contain an entry for input '%s'", flake.lockedRef, path.front()); + + return InputInfo{.lockedRef = lock->lockedRef, .isFlake = decl->isFlake, .buildTime = decl->buildTime}; + } + + /* Delegate to the child that governs this path. */ + auto key = findGoverningKey(path, true); + if (!key) { + if (!lock) + return std::nullopt; + /* An override entry below an input that we know nothing + about. */ + return InputInfo{.lockedRef = lock->lockedRef}; + } + + auto child = getChild(state, *key); + InputAttrPath rest(path.begin() + key->size(), path.end()); + + if (lock) { + /* An override entry in our lock file takes precedence + over the child's resolution; we only consult the child + for the input's metadata (such as whether it's a + flake). */ + std::optional childInfo; + try { + childInfo = child->findInput(state, rest); + } catch (Error &) { + } + return InputInfo{ + .lockedRef = lock->lockedRef, + .isFlake = !childInfo || childInfo->isFlake, + .buildTime = childInfo && childInfo->buildTime, + }; + } + + auto info = child->findInput(state, rest); + + if (info && info->parentInputAttrPath) { + /* Translate the parent path from child-relative to our + root. */ + InputAttrPath parent(*key); + parent.insert(parent.end(), info->parentInputAttrPath->begin(), info->parentInputAttrPath->end()); + info->parentInputAttrPath = std::move(parent); + } + + return info; + } + + SourcePath getSourcePath(EvalState & state, const InputAttrPath & inputAttrPath) const override + { + if (inputAttrPath.empty()) + /* The root flake. */ + return flake.path.parent(); + + if (auto lock = lockFile.findLock(inputAttrPath)) { + { + auto sourcePath(lock->sourcePath.lock()); + if (*sourcePath) + return **sourcePath; + } + + /* Note: we fetch without holding the `sourcePath` lock, + so concurrent calls don't get serialized. Racing + fetches of the same input are harmless since they + produce the same path. */ + /* Note: `lockedRef` is a copy since `mountInput()` may + modify the input (e.g. adding a `narHash` + attribute). */ + auto lockedRef = lock->lockedRef; + auto accessor = + state.inputCache + ->getAccessor(state.fetchSettings, *state.store, lockedRef.input, fetchers::UseRegistries::No) + .accessor; + auto res = state.storePath(state.mountInput(lockedRef.input, lock->lockedRef.input, accessor, true, true)) + / CanonPath(lockedRef.subdir); + + *lock->sourcePath.lock() = res; + + return res; + } + + if (inputAttrPath.size() == 1) { + auto decl = get(flake.inputs, inputAttrPath.front()); + if (decl && decl->follows) + throw Error( + "input attribute path '%s' contains unresolved 'follows' input '%s'", + printInputAttrPath(inputAttrPath), + printInputAttrPath(inputAttrPath)); + if (decl && decl->ref && decl->ref->input.isRelative()) { + /* Resolve relative path inputs against our own source + tree. */ + auto parentPath = flake.path.parent(); + return {parentPath.accessor, CanonPath(decl->ref->input.isRelative()->string(), parentPath.path)}; + } + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + } + + auto key = findGoverningKey(inputAttrPath, true); + if (!key) + throw Error("flake input '%s' does not exist", printInputAttrPath(inputAttrPath)); + + InputAttrPath rest(inputAttrPath.begin() + key->size(), inputAttrPath.end()); + return getChild(state, *key)->getSourcePath(state, rest); + } + + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const override + { + return lockFile.isUnlocked(fetchSettings); + } + + unsigned int version() const override + { + return 8; + } + + std::map getAllLockEntries(bool fetchTransitive) const override + { + if (fetchTransitive) + throw Error("fetching transitive lock files is not implemented yet for lock file version 8"); + + std::map res; + lockFile.getAllLockEntries(res); + return res; + } + + nlohmann::json toJSON() const override + { + return lockFile.toJSON(); + } +}; + +std::unique_ptr parseLockFileV8( + const fetchers::Settings & fetchSettings, Flake flake, const nlohmann::json & json, std::string_view path) +{ + experimentalFeatureSettings.require(Xp::LockFileV8); + + return std::make_unique(fetchSettings, std::move(flake), json, path); +} + +LockFlakeResult lockFlakeV8( + const Settings & settings, + EvalState & state, + const LockFlags & lockFlags, + Flake flake, + const LockedFlake & _oldLockFile) +{ + experimentalFeatureSettings.require(Xp::LockFileV8); + + /* The old lock file to reuse entries from. Null if the old lock + file is not a version 8 lock file (e.g. when migrating from + version 7), or if we're relocking from scratch. Note that + updating all inputs (`inputUpdates` = nullopt) ignores the old + lock file, but lock file entries can then still be *copied* + from dependencies' own lock files. */ + const LockFileV8 * oldLockFile = nullptr; + if (auto old = dynamic_cast(&_oldLockFile); + old && !lockFlags.recreateLockFile && lockFlags.inputUpdates) + oldLockFile = &old->lockFile; + + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; + + std::set overridesUsed; + std::set updatesUsed; + std::set explicitCliOverrides; + + /* Apply command line overrides as if they were overrides declared + by the top-level flake (`inputs.foo.inputs.bar.url = ...`), + creating intermediate override entries as needed. They + overwrite any conflicting override in `flake.nix` ("outermost + override wins"). Note: overrides of inputs that don't exist at + the top level are left unapplied so the caller can warn about + them. */ + for (auto & [path, ref] : lockFlags.inputOverrides) { + auto input = get(flake.inputs, path.get().front()); + if (!input) + continue; + for (auto & elem : std::views::drop(path.get(), 1)) + input = &input->overrides[elem]; + input->ref = ref; + input->follows = std::nullopt; + overridesUsed.insert(path); + explicitCliOverrides.insert(path); + } + + LockFileV8 newLockFile; + + std::vector parents; + + std::function + computeLocks; + + computeLocks = [&]( + /* The declared inputs of the flake being + locked (from its flake.nix). */ + const FlakeInputs & flakeInputs, + /* The lock file being computed for this + flake. */ + LockFileV8 & output, + /* The absolute input attribute path of this + flake (empty for the top-level flake). */ + const InputAttrPath & absPrefix, + /* The old locks, if any, from which locks can + be copied. */ + const LockFileV8 * oldLocks, + /* The path of this flake's `flake.nix`. */ + const SourcePath & sourcePath) { + debug("computing lock file entries for '%s'", printInputAttrPath(absPrefix)); + + /* Compute the lock for a single input or override declared by + this flake. `relPath` is relative to this flake. Returns + std::nullopt for inputs that are not stored in the lock + file ('follows' and relative path inputs). */ + auto createLock = [&](const NonEmptyInputAttrPath & relPath, + const FlakeInput & input) -> std::optional { + InputAttrPath absPath(absPrefix); + absPath.insert(absPath.end(), relPath.get().begin(), relPath.get().end()); + auto nonEmptyAbsPath = *NonEmptyInputAttrPath::make(absPath); + auto absPathS = printInputAttrPath(absPath); + debug("computing input '%s'", absPathS); + + try { + updatesUsed.insert(absPath); + + if (input.follows) { + /* 'follows' inputs are not stored in the lock + file; they are resolved at evaluation time from + the flake.nix files. */ + if (!input.overrides.empty()) + throw Error( + "input '%s' has both 'follows' and overrides for its inputs, which is not supported by lock file version 8", + absPathS); + return std::nullopt; + } + + auto ref = input.ref.value_or( + FlakeRef::fromAttrs( + state.fetchSettings, {{"type", "indirect"}, {"id", std::string(relPath.inputName())}})); + + if (auto relativePath = ref.input.isRelative()) { + /* Relative path inputs (e.g. 'path:./foo') are + not stored in the lock file, since they change + along with the flake that declares them. If + they're flakes, they must have a lock file of + their own, which is used at evaluation time. */ + SourcePath resolved{ + sourcePath.accessor, CanonPath(relativePath->string(), sourcePath.path.parent().value())}; + if (input.isFlake && !(resolved / "flake.lock").pathExists()) + throw Error( + "relative path input '%s' does not have a lock file; run 'nix flake lock %s' to create it", + absPathS, + resolved); + return std::nullopt; + } + + auto explicitUpdate = lockFlags.inputUpdates && lockFlags.inputUpdates->count(nonEmptyAbsPath); + + auto oldLock = oldLocks ? get(oldLocks->locks, relPath) : nullptr; + + if (oldLock && !explicitUpdate && oldLock->originalRef.canonicalize() == ref.canonicalize()) { + /* Copy the input from the old lock file since its + flakeref didn't change. */ + + /* Check whether an explicit update of an input + *below* this one is requested. */ + bool mustRefetch = false; + if (lockFlags.inputUpdates) { + auto lb = lockFlags.inputUpdates->lower_bound(nonEmptyAbsPath); + mustRefetch = lb != lockFlags.inputUpdates->end() && lb->get().size() > absPath.size() + && std::equal(absPath.begin(), absPath.end(), lb->get().begin()); + } + + /* If so, and this input's transitive inputs are + locked here (because it has no lock file of its + own), refetch it and recompute its nested + `locks`. Otherwise the update path doesn't + match anything we can update, and the caller + will warn about it. */ + if (!mustRefetch || !oldLock->locks) { + debug("keeping existing input '%s'", absPathS); + return oldLock->clone(); + } + + auto inputFlake = getFlake(state, oldLock->lockedRef, useRegistriesInputs, absPath, true); + + LockFileV8::Lock lock(oldLock->originalRef, oldLock->lockedRef); + *lock.sourcePath.lock() = inputFlake.path.parent(); + lock.locks = std::make_unique(); + computeLocks(inputFlake.inputs, *lock.locks, absPath, oldLock->locks.get(), inputFlake.path); + return lock; + } + + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", absPathS); + + if (!lockFlags.allowUnlocked && !ref.input.isLocked(state.fetchSettings)) + throw Error("cannot update unlocked flake input '%s' in pure mode", absPathS); + + auto useRegistriesInput = + explicitCliOverrides.contains(nonEmptyAbsPath) ? fetchers::UseRegistries::All : useRegistriesInputs; + + if (input.isFlake) { + auto inputFlake = getFlake(state, ref, useRegistriesInput, absPath, true); + + warnRegistry(absPath, ref, inputFlake.resolvedRef, flake.path); + + LockFileV8::Lock lock(ref, inputFlake.lockedRef); + *lock.sourcePath.lock() = inputFlake.path.parent(); + + /* If the input doesn't have a lock file of its + own, lock its transitive inputs here, in the + `locks` attribute of this entry. */ + if (!inputFlake.lockFilePath().pathExists()) { + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(ref); + Finally cleanup([&]() { parents.pop_back(); }); + + lock.locks = std::make_unique(); + computeLocks( + inputFlake.inputs, + *lock.locks, + absPath, + oldLock && oldLock->locks ? oldLock->locks.get() : nullptr, + inputFlake.path); + } + + return lock; + } else { + auto cachedInput = + state.inputCache->getAccessor(state.fetchSettings, *state.store, ref.input, useRegistriesInput); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), ref.subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), ref.subdir); + + warnRegistry(absPath, ref, resolvedRef, flake.path); + + /* Note: `mountInput()` adds a NAR hash to + `lockedRef.input` if it doesn't have one. */ + auto storePath = + state.storePath(state.mountInput(lockedRef.input, ref.input, cachedInput.accessor, true, true)); + + LockFileV8::Lock lock(ref, lockedRef); + *lock.sourcePath.lock() = storePath; + return lock; + } + + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", absPathS); + throw; + } + }; + + for (auto & [id, input] : flakeInputs) { + auto relPath = NonEmptyInputAttrPath::append({}, id); + + if (auto lock = createLock(relPath, input)) + output.locks.emplace(relPath, std::move(*lock)); + + /* Store the overrides declared by this flake for the + transitive inputs of this input + (e.g. `inputs.foo.inputs.bar.url = ...`), keyed by + their path relative to this flake. */ + [&](this const auto & recurse, const NonEmptyInputAttrPath & prefix, const FlakeInput & input) -> void { + for (auto & [id2, override] : input.overrides) { + auto relPath2 = NonEmptyInputAttrPath::append(prefix, id2); + if (override.follows) { + /* 'follows' overrides are not stored; they + are resolved at evaluation time from the + flake.nix files. */ + if (!override.overrides.empty()) + throw Error( + "input '%s' has both 'follows' and overrides for its inputs, which is not supported by lock file version 8", + printInputAttrPath(relPath2)); + continue; + } + if (override.ref) + if (auto lock = createLock(relPath2, override)) + output.locks.emplace(relPath2, std::move(*lock)); + recurse(relPath2, override); + } + }(relPath, input); + } + }; + + computeLocks(flake.inputs, newLockFile, {}, oldLockFile, flake.path); + + return { + .lockedFlake = std::make_unique(std::move(flake), std::move(newLockFile)), + .overridesUsed = std::move(overridesUsed), + .updatesUsed = std::move(updatesUsed), + }; +} + +} // namespace nix::flake diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc deleted file mode 100644 index c44ae77dff91..000000000000 --- a/src/libflake/lockfile.cc +++ /dev/null @@ -1,433 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nix/fetchers/fetch-settings.hh" -#include "nix/flake/lockfile.hh" -#include "nix/util/strings.hh" -#include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" -#include "nix/flake/flakeref.hh" -#include "nix/store/path.hh" -#include "nix/util/ansicolor.hh" -#include "nix/util/error.hh" -#include "nix/util/fmt.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/logging.hh" -#include "nix/util/ref.hh" -#include "nix/util/types.hh" -#include "nix/util/util.hh" - -namespace nix { -class Store; -} // namespace nix - -namespace nix::flake { - -static FlakeRef -getFlakeRef(const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) -{ - auto i = json.find(attr); - if (i != json.end()) { - auto attrs = fetchers::jsonToAttrs(*i); - // FIXME: remove when we drop support for version 5. - if (info) { - auto j = json.find(info); - if (j != json.end()) { - for (auto k : fetchers::jsonToAttrs(*j)) - attrs.insert_or_assign(k.first, k.second); - } - } - return FlakeRef::fromAttrs(fetchSettings, attrs); - } - - throw Error("attribute '%s' missing in lock file", attr); -} - -LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann::json & json) - : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) - , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) - , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) - , parentInputAttrPath( - json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) -{ - if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { - if (lockedRef.input.getNarHash()) - warn( - "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " - "This is not reproducible and will break after garbage collection or when shared.", - lockedRef.to_string()); - else - throw Error( - "Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", - fetchers::attrsToJSON(lockedRef.input.toAttrs())); - } - - // For backward compatibility, lock file entries are implicitly final. - assert(!lockedRef.input.attrs.contains("__final")); - lockedRef.input.attrs.insert_or_assign("__final", Explicit(true)); -} - -StorePath LockedNode::computeStorePath(Store & store) const -{ - return lockedRef.input.computeStorePath(store); -} - -static std::shared_ptr -doFind(const ref & root, const InputAttrPath & path, std::vector & visited) -{ - auto pos = root; - - auto found = std::find(visited.cbegin(), visited.cend(), path); - - if (found != visited.end()) { - std::vector cycle; - std::transform(found, visited.cend(), std::back_inserter(cycle), printInputAttrPath); - cycle.push_back(printInputAttrPath(path)); - throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); - } - visited.push_back(path); - - for (auto & elem : path) { - if (auto i = get(pos->inputs, elem)) { - if (auto node = std::get_if<0>(&*i)) - pos = *node; - else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = doFind(root, *follows, visited)) - pos = ref(p); - else - return {}; - } - } else - return {}; - } - - return pos; -} - -std::shared_ptr LockFile::findInput(const InputAttrPath & path) -{ - std::vector visited; - return doFind(root, path, visited); -} - -LockFile::LockFile(const fetchers::Settings & fetchSettings, std::string_view contents, std::string_view path) -{ - auto json = [=] { - try { - return nlohmann::json::parse(contents); - } catch (const nlohmann::json::parse_error & e) { - throw Error("Could not parse '%s': %s", path, e.what()); - } - }(); - auto version = json.value("version", 0); - if (version < 5 || version > 7) - throw Error("lock file '%s' has unsupported version %d", path, version); - - std::string rootKey = json["root"]; - std::map> nodeMap{{rootKey, root}}; - - [&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) { - if (jsonNode.find("inputs") == jsonNode.end()) - return; - for (auto & i : jsonNode["inputs"].items()) { - if (i.value().is_array()) { // FIXME: remove, obsolete - InputAttrPath path; - for (auto & j : i.value()) - path.push_back(j); - node.inputs.insert_or_assign(i.key(), path); - } else { - std::string inputKey = i.value(); - auto k = nodeMap.find(inputKey); - if (k == nodeMap.end()) { - auto & nodes = json["nodes"]; - auto jsonNode2 = nodes.find(inputKey); - if (jsonNode2 == nodes.end()) - throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(fetchSettings, *jsonNode2); - k = nodeMap.insert_or_assign(inputKey, input).first; - getInputs(*input, *jsonNode2); - } - if (auto child = k->second.dynamic_pointer_cast()) - node.inputs.insert_or_assign(i.key(), ref(child)); - else - // FIXME: replace by follows node - throw Error("lock file contains cycle to root node"); - } - } - }(*root, json["nodes"][rootKey]); - - // FIXME: check that there are no cycles in version >= 7. Cycles - // between inputs are only possible using 'follows' indirections. - // Once we drop support for version <= 6, we can simplify the code - // a bit since we don't need to worry about cycles. -} - -std::pair LockFile::toJSON() const -{ - nlohmann::json nodes; - KeyMap nodeKeys; - boost::unordered_flat_set keys; - - auto dumpNode = [&](this auto & dumpNode, std::string key, ref node) -> std::string { - auto k = nodeKeys.find(node); - if (k != nodeKeys.end()) - return k->second; - - if (!keys.insert(key).second) { - for (int n = 2;; ++n) { - auto k = fmt("%s_%d", key, n); - if (keys.insert(k).second) { - key = k; - break; - } - } - } - - nodeKeys.insert_or_assign(node, key); - - auto n = nlohmann::json::object(); - - if (!node->inputs.empty()) { - auto inputs = nlohmann::json::object(); - for (auto & i : node->inputs) { - if (auto child = std::get_if<0>(&i.second)) { - inputs[i.first] = dumpNode(i.first, *child); - } else if (auto follows = std::get_if<1>(&i.second)) { - auto arr = nlohmann::json::array(); - for (auto & x : *follows) - arr.push_back(x); - inputs[i.first] = std::move(arr); - } - } - n["inputs"] = std::move(inputs); - } - - if (auto lockedNode = node.dynamic_pointer_cast()) { - n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); - n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); - if (!lockedNode->isFlake) - n["flake"] = false; - if (lockedNode->buildTime) - n["buildTime"] = true; - if (lockedNode->parentInputAttrPath) - n["parent"] = *lockedNode->parentInputAttrPath; - } - - nodes[key] = std::move(n); - - return key; - }; - - nlohmann::json json; - json["version"] = 7; - json["root"] = dumpNode("root", root); - json["nodes"] = std::move(nodes); - - return {json, std::move(nodeKeys)}; -} - -std::pair LockFile::to_string() const -{ - auto [json, nodeKeys] = toJSON(); - return {json.dump(2), std::move(nodeKeys)}; -} - -std::ostream & operator<<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJSON().first.dump(2); - return stream; -} - -std::optional LockFile::isUnlocked(const fetchers::Settings & fetchSettings) const -{ - std::set> nodes; - - [&](this const auto & visit, ref node) { - if (!nodes.insert(node).second) - return; - for (auto & i : node->inputs) - if (auto child = std::get_if<0>(&i.second)) - visit(*child); - }(root); - - /* Return whether the input is either locked, or, if - `allow-dirty-locks` is enabled, it has a NAR hash. In the - latter case, we can verify the input but we may not be able to - fetch it from anywhere. */ - auto isConsideredLocked = [&](const fetchers::Input & input) { - return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); - }; - - for (auto & i : nodes) { - if (i == ref(root)) - continue; - auto node = i.dynamic_pointer_cast(); - if (node && (!isConsideredLocked(node->lockedRef.input) || !node->lockedRef.input.isFinal()) - && !node->lockedRef.input.isRelative()) - return node->lockedRef; - } - - return {}; -} - -bool LockFile::operator==(const LockFile & other) const -{ - // FIXME: slow - return toJSON().first == other.toJSON().first; -} - -InputAttrPath parseInputAttrPath(std::string_view s) -{ - InputAttrPath path; - - for (auto & elem : tokenizeString>(s, "/")) { - if (!std::regex_match(elem, flakeIdRegex)) - throw UsageError("invalid flake input attribute path element '%s'", elem); - path.push_back(elem); - } - - return path; -} - -std::optional NonEmptyInputAttrPath::parse(std::string_view s) -{ - auto path = parseInputAttrPath(s); - return make(std::move(path)); -} - -std::optional NonEmptyInputAttrPath::make(InputAttrPath path) -{ - if (path.empty()) - return std::nullopt; - return NonEmptyInputAttrPath{std::move(path)}; -} - -std::map LockFile::getAllInputs() const -{ - std::set> done; - std::map res; - - [&](this const auto & recurse, const InputAttrPath & prefix, ref node) { - if (!done.insert(node).second) - return; - - for (auto & [id, input] : node->inputs) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(id); - res.emplace(inputAttrPath, input); - if (auto child = std::get_if<0>(&input)) - recurse(inputAttrPath, *child); - } - }({}, root); - - return res; -} - -static std::string describe(const FlakeRef & flakeRef) -{ - auto s = fmt("'%s'", flakeRef.to_string(true)); - - if (auto lastModified = flakeRef.input.getLastModified()) - s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); - - return s; -} - -std::ostream & operator<<(std::ostream & stream, const Node::Edge & edge) -{ - if (auto node = std::get_if<0>(&edge)) - stream << describe((*node)->lockedRef); - else if (auto follows = std::get_if<1>(&edge)) - stream << fmt("follows '%s'", printInputAttrPath(*follows)); - return stream; -} - -static bool equals(const Node::Edge & e1, const Node::Edge & e2) -{ - if (auto n1 = std::get_if<0>(&e1)) - if (auto n2 = std::get_if<0>(&e2)) - return (*n1)->lockedRef == (*n2)->lockedRef; - if (auto f1 = std::get_if<1>(&e1)) - if (auto f2 = std::get_if<1>(&e2)) - return *f1 == *f2; - return false; -} - -std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) -{ - auto oldFlat = oldLocks.getAllInputs(); - auto newFlat = newLocks.getAllInputs(); - - auto i = oldFlat.begin(); - auto j = newFlat.begin(); - std::string res; - - while (i != oldFlat.end() || j != newFlat.end()) { - if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { - res += fmt( - "• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", printInputAttrPath(j->first), j->second); - ++j; - } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { - res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first)); - ++i; - } else { - if (!equals(i->second, j->second)) { - res += - fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", - printInputAttrPath(i->first), - i->second, - j->second); - } - ++i; - ++j; - } - } - - return res; -} - -void LockFile::check() -{ - auto inputs = getAllInputs(); - - for (auto & [inputAttrPath, input] : inputs) { - if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !findInput(*follows)) - throw Error( - "input '%s' follows a non-existent input '%s'", - printInputAttrPath(inputAttrPath), - printInputAttrPath(*follows)); - } - } -} - -void check(); - -std::string printInputAttrPath(const InputAttrPath & path) -{ - return concatStringsSep("/", path); -} - -} // namespace nix::flake diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 1bf6f7e5fba6..8db1aaa858fe 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -41,10 +41,13 @@ endforeach sources = files( 'config.cc', + 'diff.cc', 'flake-primops.cc', 'flake.cc', 'flakeref.cc', - 'lockfile.cc', + 'input-attr-path.cc', + 'lockfile-v7.cc', + 'lockfile-v8.cc', 'provenance.cc', 'settings.cc', 'url-name.cc', diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 674deec7f85c..cad83f70d4b4 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -27,7 +27,7 @@ void MissingExperimentalFeature::anchor() {} * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::CNSA); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::LockFileV8); constexpr std::array xpFeatureDetails = {{ { @@ -317,6 +317,17 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::LockFileV8, + .name = "lock-file-v8", + .description = R"( + Enable support for version 8 ("sparse") flake lock files, which + only store the locks of a flake's immediate inputs. Use the + [`lock-file-format`](@docroot@/command-ref/conf-file.md#conf-lock-file-format) + setting to create version 8 lock files. + )", + .trackingUrl = "", + }, }}; static_assert( diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index f20024f0387e..4eed26bcfb88 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -42,6 +42,7 @@ enum struct ExperimentalFeature { WasmDerivations, Provenance, CNSA, + LockFileV8, }; extern std::set stabilizedFeatures; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ec190d26f6fe..5295a963bf35 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -641,7 +641,7 @@ struct CmdDevelop : Common, MixEnvironment auto nixpkgsLockFlags = lockFlags; nixpkgsLockFlags.inputOverrides = {}; - nixpkgsLockFlags.inputUpdates = {}; + nixpkgsLockFlags.inputUpdates = std::set(); auto nixpkgs = defaultNixpkgsFlakeRef(); if (auto * i = dynamic_cast(&*installable)) diff --git a/src/nix/flake-command.hh b/src/nix/flake-command.hh index ae52bcabb912..04e0801cfb30 100644 --- a/src/nix/flake-command.hh +++ b/src/nix/flake-command.hh @@ -17,7 +17,7 @@ public: FlakeRef getFlakeRef(); - flake::LockedFlake lockFlake(); + std::unique_ptr lockFlake(); std::vector getFlakeRefsForCompletion() override; }; diff --git a/src/nix/flake-diff-locks.cc b/src/nix/flake-diff-locks.cc new file mode 100644 index 000000000000..448e203998f0 --- /dev/null +++ b/src/nix/flake-diff-locks.cc @@ -0,0 +1,95 @@ +#include "flake-command.hh" +#include "nix/expr/eval.hh" +#include "nix/fetchers/fetch-settings.hh" + +#include +#include + +namespace nix { + +using namespace flake; + +struct CmdFlakeDiffLocks : EvalCommand +{ + std::string oldFlakeUrl, newFlakeUrl = "."; + + bool transitive = false; + + CmdFlakeDiffLocks() + { + expectArgs( + {.label = "old-flake", + .handler = {&oldFlakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); + + expectArgs( + {.label = "new-flake", + .optional = true, + .handler = {&newFlakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); + + addFlag({ + .longName = "transitive", + .description = "Include the transitive locks of inputs that have a lock file of their own. " + "This may require fetching those inputs.", + .handler = {&transitive, true}, + }); + } + + std::string description() override + { + return "show the differences between the lock files of two flakes"; + } + + std::string doc() override + { + return +#include "flake-diff-locks.md" + ; + } + + void run(nix::ref store) override + { + auto state = getEvalState(); + + auto readLockedFlake = [&](const std::string & flakeUrl) { + auto flake = getFlake( + *state, + parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()), + fetchers::UseRegistries::All, + false); + + nlohmann::json json; + + auto lockFilePath = flake.lockFilePath(); + if (lockFilePath.pathExists()) { + try { + json = nlohmann::json::parse(lockFilePath.readFile()); + } catch (const nlohmann::json::parse_error & e) { + throw Error("Could not parse '%s': %s", lockFilePath, e.what()); + } + } else + warn("flake '%s' does not have a lock file", flake.originalRef); + + return parseLockFile(state->fetchSettings, std::move(flake), json, fmt("%s", lockFilePath)); + }; + + auto oldLockedFlake = readLockedFlake(oldFlakeUrl); + auto newLockedFlake = readLockedFlake(newFlakeUrl); + + auto diff = diffLockedFlakes(*oldLockedFlake, *newLockedFlake, transitive); + + if (diff.empty()) + logger->cout("No changes."); + else + logger->cout("%s", chomp(diff)); + } +}; + +static auto rCmdFlakeDiffLocks = registerCommand2({"flake", "diff-locks"}); + +} // namespace nix diff --git a/src/nix/flake-diff-locks.md b/src/nix/flake-diff-locks.md new file mode 100644 index 000000000000..cad7f46bbfd4 --- /dev/null +++ b/src/nix/flake-diff-locks.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Show how the locked inputs of two versions of a flake differ: + + ```console + # nix flake diff-locks github:NixOS/nix/2.28.0 github:NixOS/nix/2.29.0 + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/48d12d5' (2024-12-16) + → 'github:NixOS/nixpkgs/adaa24f' (2025-05-13) + ``` + +* Show what has changed in the lock file of the flake in the worktree relative to a previous Git revision: + + ```console + # nix flake diff-locks '.?rev=26842787496f2293c676fb36db38dacfd63497e0' + ``` + + or relative to a Git ref: + + ```console + # nix flake diff-locks '.?ref=HEAD' + ``` + +# Description + +This command shows the differences between the lock files of two flakes *old-flake* and *new-flake*: inputs that were added, removed or updated. *new-flake* defaults to the flake in the current directory. +The flakes do not need to use the same lock file format version; a change in the lock file version is also reported. + +By default, only the locks contained in the two flakes' own lock files are compared. +With `--transitive`, inputs that have a lock file of their own are fetched in order to include their transitive locks in the comparison. + +)"" diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index f4b944fcea53..33c35ed3ad77 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -1,5 +1,4 @@ #include "flake-command.hh" -#include "nix/fetchers/fetch-to-store.hh" #include "nix/util/thread-pool.hh" #include "nix/store/filetransfer.hh" #include "nix/util/exit.hh" @@ -24,46 +23,48 @@ struct CmdFlakePrefetchInputs : FlakeCommand void run(nix::ref store) override { - using namespace nix::flake; auto flake = lockFlake(); - ThreadPool pool{fileTransferSettings.httpConnections}; + auto state = getEvalState(); - struct State - { - std::set done; - }; + /* Gather the attribute paths of all transitive inputs, + skipping build-time inputs and their dependencies. */ + std::vector> inputs; - Sync state_; + flake->visit(*state, [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + auto inputInfo = std::get_if(&input); - std::atomic nrFailed{0}; + /* Skip "follows" inputs and build-time inputs (and their + dependencies). */ + if (!inputInfo || inputInfo->buildTime) + return false; + + /* Skip the root flake, which we've fetched already. */ + if (!inputAttrPath.empty()) + inputs.emplace_back(inputAttrPath, inputInfo->lockedRef); - auto visit = [&](this const auto & visit, const Node & node) { - if (!state_.lock()->done.insert(&node).second) - return; + return true; + }); + + /* Fetch the inputs in parallel. */ + ThreadPool pool{fileTransferSettings.httpConnections}; - if (auto lockedNode = dynamic_cast(&node)) { - if (lockedNode->buildTime) - return; + std::atomic nrFailed{0}; + + for (auto & [inputAttrPath, lockedRef] : inputs) { + pool.enqueue([&, inputAttrPath, lockedRef]() { try { - Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); - auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first; - if (!evalSettings.lazyTrees) - fetchToStore( - fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); + Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedRef)); + /* Note: when lazy trees are disabled, this also + copies the input to the store (via + `EvalState::mountInput()`). */ + flake->getSourcePath(*state, inputAttrPath); } catch (Error & e) { printError("%s", e.what()); nrFailed++; } - } - - for (auto & [inputName, input] : node.inputs) { - if (auto inputNode = std::get_if<0>(&input)) - pool.enqueue(std::bind(visit, **inputNode)); - } - }; - - pool.enqueue(std::bind(visit, *flake.lockFile.root)); + }); + } pool.process(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3deddb7dc6e6..0186cb637750 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -56,7 +56,7 @@ FlakeRef FlakeCommand::getFlakeRef() return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); // FIXME } -flake::LockedFlake FlakeCommand::lockFlake() +std::unique_ptr FlakeCommand::lockFlake() { return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } @@ -106,11 +106,11 @@ struct CmdFlakeUpdate : FlakeCommand inputToUpdate); throw e; } - if (lockFlags.inputUpdates.contains(*inputAttrPath)) + if (lockFlags.inputUpdates->contains(*inputAttrPath)) warn( "Input '%s' was specified multiple times. You may have done this by accident.", printInputAttrPath(*inputAttrPath)); - lockFlags.inputUpdates.insert(*inputAttrPath); + lockFlags.inputUpdates->insert(*inputAttrPath); } }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { @@ -133,9 +133,11 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { fetchSettings.tarballTtl = 0; - auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = updateAll; + /* If no specific inputs are given, update all inputs. */ + if (lockFlags.inputUpdates->empty()) + lockFlags.inputUpdates = std::nullopt; + lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; lockFlags.requireLockable = false; @@ -195,7 +197,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON { lockFlags.requireLockable = false; auto lockedFlake = lockFlake(); - auto & flake = lockedFlake.flake; + auto & flake = lockedFlake->flake; /* Hack to show the store path if available. */ std::optional storePath; @@ -227,8 +229,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; if (storePath) j["path"] = store->printStorePath(*storePath); - j["locks"] = lockedFlake.lockFile.toJSON().first; - if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) + j["locks"] = lockedFlake->toJSON(); + if (auto fingerprint = lockedFlake->getFingerprint(*store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); printJSON(j); } else { @@ -249,41 +251,56 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) + if (auto fingerprint = lockedFlake->getFingerprint(*store, fetchSettings)) logger->cout( ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false)); - if (!lockedFlake.lockFile.root->inputs.empty()) - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + /* Gather the inputs into a tree, since we need to know + the children of a node before we can print it. */ + struct TreeNode + { + std::optional> input; + std::map children; + }; - std::set> visited{lockedFlake.lockFile.root}; + TreeNode root; + + lockedFlake->visit(*getEvalState(), [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + if (!inputAttrPath.empty()) { + auto * node = &root; + for (auto & elem : inputAttrPath) + node = &node->children[elem]; + node->input = input; + } + return true; + }); - [&](this const auto & recurse, const flake::Node & node, const std::string & prefix) -> void { - for (const auto & [last, input] : markLast(node.inputs)) { - if (auto lockedNode = std::get_if<0>(&input.second)) { + if (!root.children.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + + [&](this const auto & recurse, const TreeNode & node, const std::string & prefix) -> void { + for (const auto & [last, child] : markLast(node.children)) { + if (auto inputInfo = std::get_if(&*child.second.input)) { std::string lastModifiedStr = ""; - if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + if (auto lastModified = inputInfo->lockedRef.input.getLastModified()) lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); logger->cout( "%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), - input.first, - (*lockedNode)->lockedRef.to_string(true), + child.first, + inputInfo->lockedRef.to_string(true), lastModifiedStr); - bool firstVisit = visited.insert(*lockedNode).second; - - if (firstVisit) - recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); - } else if (auto follows = std::get_if<1>(&input.second)) { + recurse(child.second, prefix + (last ? treeNull : treeLine)); + } else if (auto follows = std::get_if(&*child.second.input)) { logger->cout( "%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), - input.first, + child.first, flake::printInputAttrPath(*follows)); } } - }(*lockedFlake.lockFile.root, ""); + }(root, ""); } } }; @@ -370,7 +387,7 @@ struct CmdFlakeCheck : FlakeCommand, MixPrintOutPaths, MixOutLinkBase, MixFlakeS auto state = getEvalState(); lockFlags.applyNixConfig = true; - auto flake = std::make_shared(lockFlake()); + std::shared_ptr flake(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); @@ -812,45 +829,42 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs StorePathSet sources; - auto storePath = dryRun ? flake.flake.lockedRef.input.computeStorePath(*store) - : std::get(flake.flake.lockedRef.input.fetchToStore(fetchSettings, *store)); - - sources.insert(storePath); - - // FIXME: use graph output, handle cycles. - auto traverse = - [&, json = json, dryRun = dryRun](this const auto & self, const flake::Node & node) -> nlohmann::json { - nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); - for (auto & [inputName, input] : node.inputs) { - if (auto inputNode = std::get_if<0>(&input)) { - std::optional storePath; - if (!(*inputNode)->lockedRef.input.isRelative()) { - storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : std::get( - (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store)); - sources.insert(*storePath); - } - if (json) { - auto & jsonObj3 = jsonObj2[inputName]; - if (storePath) - jsonObj3["path"] = store->printStorePath(*storePath); - jsonObj3["inputs"] = self(**inputNode); - } else - self(**inputNode); - } - } - return jsonObj2; + nlohmann::json jsonRoot; + + /* Return the JSON object for the input denoted by + `inputAttrPath`, creating it if necessary. */ + auto getJsonObj = [&](const flake::InputAttrPath & inputAttrPath) -> nlohmann::json & { + auto * jsonObj = &jsonRoot; + for (auto & elem : inputAttrPath) + jsonObj = &(*jsonObj)["inputs"][elem]; + return *jsonObj; }; - if (json) { - nlohmann::json jsonRoot = { - {"path", store->printStorePath(storePath)}, - {"inputs", traverse(*flake.lockFile.root)}, - }; + flake->visit(*getEvalState(), [&](const flake::InputAttrPath & inputAttrPath, const auto & input) { + /* Skip "follows" inputs; their targets are visited under + their own paths. */ + auto inputInfo = std::get_if(&input); + if (!inputInfo) + return false; + + std::optional storePath; + if (!inputInfo->lockedRef.input.isRelative()) { + storePath = dryRun + ? inputInfo->lockedRef.input.computeStorePath(*store) + : std::get(inputInfo->lockedRef.input.fetchToStore(fetchSettings, *store)); + sources.insert(*storePath); + } + if (json) { + auto & jsonObj = getJsonObj(inputAttrPath); + if (storePath) + jsonObj["path"] = store->printStorePath(*storePath); + jsonObj["inputs"] = nlohmann::json::object(); + } + return true; + }); + + if (json) printJSON(jsonRoot); - } else { - traverse(*flake.lockFile.root); - } if (!dryRun && dstUri) { ref dstStore = openStore(StoreReference{*dstUri}); @@ -918,7 +932,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON, MixFlakeSchemas throw UsageError("The '--drv-paths' flag requires '--json'."); auto state = getEvalState(); - auto flake = make_ref(lockFlake()); + std::shared_ptr flake(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); diff --git a/src/nix/meson.build b/src/nix/meson.build index 9515101546c0..c854206dc19a 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -109,7 +109,7 @@ nix_sources = [ config_priv_h ] + files( 'edit.cc', 'env.cc', 'eval.cc', - 'flake-prefetch-inputs.cc', + 'flake-diff-locks.cc', 'flake-prefetch-inputs.cc', 'flake.cc', 'formatter.cc',