diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d2022aeb09d0..5aa1e735ca67 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -133,7 +133,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() std::pair InstallableFlake::toValue(EvalState & state) { - return {&getCursor(state)->forceValue(), noPos}; + return {&getCursor(state)->getValue(), noPos}; } std::vector InstallableFlake::getAttrPaths(bool useDefaultAttrPath, ref inventory) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index d2f041a06bf8..77e48651d420 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -785,7 +785,8 @@ StorePath AttrCursor::forceDerivation() root->state.store->addTempRoot(drvPath); if (!root->state.store->isValidPath(drvPath)) { /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ + been garbage-collected. So force it to be regenerated. + FIXME: we don't need to do this if the output paths are already valid or can be substituted. */ aDrvPath->forceValue(); root->state.waitForPath(drvPath); if (!root->state.store->isValidPath(drvPath)) diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index df237d704b23..349d63b69b7c 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -120,8 +120,6 @@ private: AttrKey getKey(); - Value & getValue(); - /** * If `cachedValue` is unset, try to initialize it from the * database. It is not an error if it does not exist. Throw a @@ -180,6 +178,14 @@ public: bool isDerivation(); + /** + * Return the value, which may be in a thunk state. + */ + Value & getValue(); + + /** + * Force and return the value. + */ Value & forceValue(); /** diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index b0ae6c6ecb0f..7deda6ae9656 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -228,7 +228,7 @@ struct StaticEvalSymbols line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure, outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet, - operator_, key, path, prefix, outputSpecified, __meta; + operator_, key, path, prefix, outputSpecified, __meta, drvAttrs; Expr::AstSymbols exprSymbols; @@ -282,6 +282,7 @@ struct StaticEvalSymbols .prefix = alloc.create("prefix"), .outputSpecified = alloc.create("outputSpecified"), .__meta = alloc.create("__meta"), + .drvAttrs = alloc.create("drvAttrs"), .exprSymbols = { .sub = alloc.create("__sub"), .lessThan = alloc.create("__lessThan"), @@ -674,6 +675,14 @@ public: */ void forceValueDeep(Value & v); + /** + * Force a value, then recursively force list elements and attributes in parallel. For derivations, we recurse into + * `drvAttrs` but no other attributes (e.g. `meta` and `passthru` are not evaluated). + * + * This function does nothing if parallel evaluation is disabled. + */ + void forceValueDeepParallel(Value & v, PosIdx pos); + /** * Force `v`, and then verify that it has the expected type. */ diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 1c89b9c4db2c..5221aafea872 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -3,6 +3,8 @@ #include "nix/store/globals.hh" #include "nix/expr/primops.hh" +#include + namespace nix { // cache line alignment to prevent false sharing @@ -103,8 +105,9 @@ void Executor::worker() return; } if (!state->queue.empty()) { - item = std::move(state->queue.begin()->second); - state->queue.erase(state->queue.begin()); + auto i = state->queue.begin(); + item = std::move(i->second); + state->queue.erase(i); break; } state.wait(wakeup); @@ -314,4 +317,77 @@ static RegisterPrimOp r_parallel({ .experimentalFeature = Xp::ParallelEval, }); +#pragma GCC diagnostic ignored "-Wswitch-enum" + +void EvalState::forceValueDeepParallel(Value & vRoot, PosIdx pos) +{ + if (!executor->enabled) + return; + + // FIXME: the pointers in this set can refer to values that been GCed and then reallocated. That's not a problem for + // correctness, since at worst it prevents background evaluation of some values. But we should probably register a + // GC hook to clear this set at GC time. + static boost::concurrent_flat_set seen; + + Executor::WorkItems work; + + auto recurse = [&](this const auto & recurse, EvalState & state, Value & v, PosIdx pos) -> void { + auto type = v.type(); + if (type == nString || type == nPath || type == nNull || type == nInt || type == nFloat || type == nBool + || type == nFailed || type == nExternal) + return; + + if (!seen.insert(&v) && &v != &vRoot) + return; + + if (type == nThunk) { + state.addWork(work, 0, [v(RootValue(&v)), pos, &state]() { state.forceValueDeepParallel(**v, pos); }); + return; + } + + switch (v.type()) { + + case nAttrs: { + + NixStringContext context; + if (state.tryAttrsToString(pos, v, context, false, false)) + return; + + if (auto aDrvPath = v.attrs()->get(s.drvPath)) { + if (aDrvPath->value->isFinished()) + return; + + if (auto aDrvAttrs = v.attrs()->get(s.drvAttrs)) + recurse(state, *aDrvAttrs->value, aDrvAttrs->pos); + + } else { + for (auto & a : *v.attrs()) + recurse(state, *a.value, a.pos); + } + + break; + } + + case nList: { + for (const auto & elem : v.listView()) + recurse(state, *elem, pos); + break; + } + + default: + break; + } + }; + + forceValue(vRoot, pos); + + recurse(*this, vRoot, pos); + + if (work.size() == 1) + // Only one work item, so we may as well do it on the current thread right away. + work[0].first(); + else + executor->spawn(std::move(work)); +} + } // namespace nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cf009515c57c..53a9f6574d16 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1420,6 +1420,9 @@ static void prim_derivationStrictGeneric(EvalState & state, const PosIdx pos, Va auto attrs = args[0]->attrs(); + /* If parallel eval is enabled, then start evaluating the entire drv graph in the background. */ + state.forceValueDeepParallel(*args[0], noPos); + /* Figure out the name first (for stack backtraces). */ auto nameAttr = state.getAttr(state.s.name, attrs, "in the attrset passed as argument to builtins.derivationStrict"); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 134417e165d6..2846cb46f174 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,41 +11,12 @@ namespace nix { using json = nlohmann::json; -#pragma GCC diagnostic ignored "-Wswitch-enum" - -static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos) -{ - state.forceValue(v, pos); - - Executor::WorkItems work; - - switch (v.type()) { - - case nAttrs: { - NixStringContext context; - if (state.tryAttrsToString(pos, v, context, false, false)) - return; - if (v.attrs()->get(state.s.outPath)) - return; - for (auto & a : *v.attrs()) - state.addWork( - work, 0, [value(RootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }); - break; - } - - default: - break; - } - - state.executor->spawn(std::move(work)); -} - // TODO: rename. It doesn't print. json printValueAsJSON( EvalState & state, bool strict, Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { if (strict && state.executor->enabled && !Executor::amWorkerThread) - parallelForceDeep(state, v, pos); + state.forceValueDeepParallel(v, pos); auto recurse = [&](this const auto & recurse, json & res, Value & v, PosIdx pos) -> void { checkInterrupt();