Allow dynamic derivation deps in inputDrvs

We use the same nested map representation we used for goals, again in
order to save space. We might someday want to combine with `inputDrvs`,
by doing `V = bool` instead of `V = std::set<OutputName>`, but we are
not doing that yet for sake of a smaller diff.

The ATerm format for Derivations also needs to be extended, in addition
to the in-memory format. To accomodate this, we added a new basic
versioning scheme, so old versions of Nix will get nice errors. (And
going forward, if the ATerm format changes again the errors will be even
better.)

`parsedStrings`, an internal function used as part of parsing
derivations in A-Term format, used to consume the final `]` but expect
the initial `[` to already be consumed. This made for what looked like
unbalanced brackets at callsites, which was confusing. Now it consumes
both which is hopefully less confusing.

As part of testing, we also created a unit test for the A-Term format for
regular non-experimental derivations too.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

doc/manual/src/protocols/derivation-aterm.md

@@ -2,8 +2,18 @@
 
 For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format.
 
-Derivations are serialised in the following format:
+Derivations are serialised in one of the following formats:
 
-```
-Derive(...)
-```
+- ```
+  Derive(...)
+  ```
+
+  For all stable derivations.
+
+- ```
+  Derive(<version-string>, ...)
+  ```
+
+  The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md):
+
+  - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

perl/lib/Nix/Store.xs

@@ -324,7 +324,7 @@ SV * derivationFromPath(char * drvPath)
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();
-            for (auto & i : drv.inputDrvs)
+            for (auto & i : drv.inputDrvs.map)
                 av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
 

src/build-remote/build-remote.cc

@@ -314,7 +314,7 @@ static int main_build_remote(int argc, char * * argv)
             //
             // 2. Changing the `inputSrcs` set changes the associated
             //    output ids, which break CA derivations
-            if (!drv.inputDrvs.empty())
+            if (!drv.inputDrvs.map.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
             optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
             auto & result = *optResult;

src/libcmd/built-path.cc

@@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+SingleDerivedPath::Built SingleBuiltPath::Built::discaredOutputPath() const
+{
+    return SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(drvPath->discaredOutputPath()),
+        .output = output.first,
+    };
+}
+
+SingleDerivedPath SingleBuiltPath::discaredOutputPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath {
+                return p;
+            },
+            [](const SingleBuiltPath::Built & b) -> SingleDerivedPath {
+                return b.discaredOutputPath();
+            },
+        }, raw()
+    );
+}
+
 nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
 {
     nlohmann::json res;

src/libcmd/built-path.hh

@@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::pair<std::string, StorePath> output;
 
+    SingleDerivedPathBuilt discaredOutputPath() const;
+
     std::string to_string(const Store & store) const;
     static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
@@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     StorePath outPath() const;
 
+    SingleDerivedPath discaredOutputPath() const;
+
     static SingleBuiltPath parse(const Store & store, std::string_view);
     nlohmann::json toJSON(const Store & store) const;
 };

src/libexpr/primops.cc

@@ -1252,15 +1252,15 @@ drvName, Bindings * attrs, Value & v)
                 state.store->computeFSClosure(d.drvPath, refs);
                 for (auto & j : refs) {
                     drv.inputSrcs.insert(j);
-                    if (j.isDerivation())
-                        drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
+                    if (j.isDerivation()) {
+                        drv.inputDrvs.map[j] = DerivedPathMap<StringSet>::ChildNode {
+                            .value = state.store->readDerivation(j).outputNames(),
+                        };
+                    }
                 }
             },
             [&](const NixStringContextElem::Built & b) {
-                if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
-                    drv.inputDrvs[p->path].insert(b.output);
-                else
-                    throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
+                drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
                 drv.inputSrcs.insert(o.path);

src/libstore/build/derivation-goal.cc

@@ -351,24 +351,37 @@ void DerivationGoal::gaveUpOnSubstitution()
     /* The inputs must be built before we can build this goal. */
     inputDrvOutputs.clear();
     if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+    {
+        std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+
+        accumDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            if (!inputNode.value.empty())
+                addWaitee(worker.makeGoal(
+                    DerivedPath::Built {
+                        .drvPath = inputDrv,
+                        .outputs = inputNode.value,
+                    },
+                    buildMode == bmRepair ? bmRepair : bmNormal));
+            for (const auto & [outputName, childNode] : inputNode.childMap)
+                accumDerivedPath(
+                    make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
+                    childNode);
+        };
+
+        for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
             /* Ensure that pure, non-fixed-output derivations don't
                depend on impure derivations. */
             if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
-                auto inputDrv = worker.evalStore.readDerivation(i.first);
+                auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
                 if (!inputDrv.type().isPure())
                     throw Error("pure derivation '%s' depends on impure derivation '%s'",
                         worker.store.printStorePath(drvPath),
-                        worker.store.printStorePath(i.first));
+                        worker.store.printStorePath(inputDrvPath));
             }
 
-            addWaitee(worker.makeGoal(
-                DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(i.first),
-                    .outputs = i.second,
-                },
-                buildMode == bmRepair ? bmRepair : bmNormal));
+            accumDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
         }
+    }
 
     /* Copy the input sources from the eval store to the build
        store. */
@@ -501,7 +514,7 @@ void DerivationGoal::inputsRealised()
                 return ia.deferred;
             },
             [&](const DerivationType::ContentAddressed & ca) {
-                return !fullDrv.inputDrvs.empty() && (
+                return !fullDrv.inputDrvs.map.empty() && (
                     ca.fixed
                     /* Can optionally resolve if fixed, which is good
                        for avoiding unnecessary rebuilds. */
@@ -515,7 +528,7 @@ void DerivationGoal::inputsRealised()
             }
         }, drvType.raw);
 
-        if (resolveDrv && !fullDrv.inputDrvs.empty()) {
+        if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
 
             /* We are be able to resolve this derivation based on the
@@ -552,11 +565,13 @@ void DerivationGoal::inputsRealised()
             return;
         }
 
-        for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
+        std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumDerivedPath;
+
+        accumDerivedPath = [&](const StorePath & depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs) {
+            auto getOutput = [&](const std::string & outputName) {
                 /* TODO (impure derivations-induced tech debt):
                    Tracking input derivation outputs statefully through the
                    goals is error prone and has led to bugs.
@@ -568,21 +583,30 @@ void DerivationGoal::inputsRealised()
                    a representation in the store, which is a usability problem
                    in itself. When implementing this logic entirely with lookups
                    make sure that they're cached. */
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
-                    worker.store.computeFSClosure(*outPath, inputPaths);
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) {
+                    return *outPath;
                 }
                 else {
                     auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
-                    auto outMapPath = outMap.find(j);
+                    auto outMapPath = outMap.find(outputName);
                     if (outMapPath == outMap.end()) {
                         throw Error(
                             "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                            worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath));
                     }
-                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                    return outMapPath->second;
                 }
-            }
-        }
+            };
+
+            for (auto & outputName : inputNode.value)
+                worker.store.computeFSClosure(getOutput(outputName), inputPaths);
+
+            for (auto & [outputName, childNode] : inputNode.childMap)
+                accumDerivedPath(getOutput(outputName), childNode);
+        };
+
+        for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map)
+            accumDerivedPath(depDrvPath, depNode);
     }
 
     /* Second, the input sources. */
@@ -1475,22 +1499,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    auto * dg = tryGetConcreteDrvGoal(waitee);
-    if (!dg) return;
+    std::optional info = tryGetConcreteDrvGoal(waitee);
+    if (!info) return;
+    const auto & [dg, drvReq] = *info;
 
-    auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
-    if (outputs == fullDrv.inputDrvs.end()) return;
+    auto & outputs = fullDrv.inputDrvs.ensureSlot(drvReq.get()).value;
+    // FIXME: need a `findSlot`
+    //if (outputs == fullDrv.inputDrvs.end()) return;
 
-    for (auto & outputName : outputs->second) {
-        auto buildResult = dg->getBuildResult(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg->drvPath),
+    for (auto & outputName : outputs) {
+        auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(dg.get().drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {
             auto i = buildResult.builtOutputs.find(outputName);
             if (i != buildResult.builtOutputs.end())
                 inputDrvOutputs.insert_or_assign(
-                    { dg->drvPath, outputName },
+                    { dg.get().drvPath, outputName },
                     i->second.outPath);
         }
     }

src/libstore/build/worker.cc

@@ -594,11 +594,14 @@ GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
     return subGoal;
 }
 
-const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
 {
     auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
-    if (!odg) return nullptr;
-    return &*odg->concreteDrvGoal;
+    if (!odg) return std::nullopt;
+    return {{
+        std::cref(*odg->concreteDrvGoal),
+        std::cref(*odg->drvReq),
+    }};
 }
 
 }

src/libstore/build/worker.hh

@@ -49,7 +49,8 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
  * we have made the function, written in `worker.cc` where all the goal
  * types are visible, and use it instead.
  */
-const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
+
+std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
 
 /**
  * A mapping used to remember for each child process to what goal it