Allow selecting derivation outputs

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,13 @@
 
 * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
   This matches the default behaviour of `nix-build`.
+
+* You can now specify which outputs of a derivation `nix` should
+  operate on using the syntax `installable^outputs`,
+  e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
+  `nix` will use the outputs specified by the derivation's
+  `meta.outputsToInstall` attribute if it exists, or all outputs
+  otherwise.
+
+  Selecting derivation outputs using the attribute selection syntax
+  (e.g. `nixpkgs#glibc.dev`) no longer works.

src/libcmd/installables.cc

@@ -464,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
     SourceExprCommand & cmd;
     RootValue v;
     std::string attrPath;
-
-    InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
-        : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
+    OutputsSpec outputsSpec;
+
+    InstallableAttrPath(
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        const std::string & attrPath,
+        OutputsSpec outputsSpec)
+        : InstallableValue(state)
+        , cmd(cmd)
+        , v(allocRootValue(v))
+        , attrPath(attrPath)
+        , outputsSpec(std::move(outputsSpec))
     { }
 
     std::string what() const override { return attrPath; }
@@ -495,9 +505,15 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
         auto drvPath = drvInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
+
         std::set<std::string> outputsToInstall;
-        for (auto & output : drvInfo.queryOutputs(false, true))
-            outputsToInstall.insert(output.first);
+
+        if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+            outputsToInstall = *outputNames;
+        else
+            for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
+                outputsToInstall.insert(output.first);
+
         res.push_back(DerivationInfo {
             .drvPath = *drvPath,
             .outputsToInstall = std::move(outputsToInstall)
@@ -578,13 +594,15 @@ InstallableFlake::InstallableFlake(
     ref<EvalState> state,
     FlakeRef && flakeRef,
     std::string_view fragment,
+    OutputsSpec outputsSpec,
     Strings attrPaths,
     Strings prefixes,
     const flake::LockFlags & lockFlags)
     : InstallableValue(state),
       flakeRef(flakeRef),
       attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
       prefixes(fragment == "" ? Strings{} : prefixes),
+      outputsSpec(std::move(outputsSpec)),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -609,14 +627,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
             for (auto & s : aOutputsToInstall->getListOfStrings())
                 outputsToInstall.insert(s);
 
-    if (outputsToInstall.empty())
+    if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
+        outputsToInstall.clear();
         if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
             for (auto & s : aOutputs->getListOfStrings())
                 outputsToInstall.insert(s);
+    }
 
     if (outputsToInstall.empty())
         outputsToInstall.insert("out");
 
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        outputsToInstall = *outputNames;
+
     auto drvInfo = DerivationInfo {
         .drvPath = std::move(drvPath),
         .outputsToInstall = std::move(outputsToInstall),
@@ -742,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             state->eval(e, *vFile);
         }
 
-        for (auto & s : ss)
-            result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
+        for (auto & s : ss) {
+            auto [prefix, outputsSpec] = parseOutputsSpec(s);
+            result.push_back(
+                std::make_shared<InstallableAttrPath>(
+                    state, *this, vFile,
+                    prefix == "." ? "" : prefix,
+                    outputsSpec));
+        }
 
     } else {
 
@@ -762,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
+                auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
                         fragment,
+                        outputsSpec,
                         getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));

src/libcmd/installables.hh

@@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
     FlakeRef flakeRef;
     Strings attrPaths;
     Strings prefixes;
+    OutputsSpec outputsSpec;
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
@@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
         ref<EvalState> state,
         FlakeRef && flakeRef,
         std::string_view fragment,
+        OutputsSpec outputsSpec,
         Strings attrPaths,
         Strings prefixes,
         const flake::LockFlags & lockFlags);

src/libexpr/flake/flakeref.cc

@@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
     return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
+{
+    auto [prefix, outputsSpec] = parseOutputsSpec(url);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
+    return {std::move(flakeRef), fragment, outputsSpec};
+}
+
 }

src/libexpr/flake/flakeref.hh

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "fetchers.hh"
+#include "path-with-outputs.hh"
 
 #include <variant>
 
@@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
+
+
 }

src/libstore/path-with-outputs.cc

@@ -1,5 +1,8 @@
 #include "path-with-outputs.hh"
 #include "store-api.hh"
+#include "nlohmann/json.hpp"
+
+#include <regex>
 
 namespace nix {
 
@@ -68,4 +71,57 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
     return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
 }
 
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
+{
+    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
+
+    std::smatch match;
+    if (!std::regex_match(s, match, regex))
+        return {s, DefaultOutputs()};
+
+    if (match[3].matched)
+        return {match[1], AllOutputs()};
+
+    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
+}
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        return "";
+
+    if (std::get_if<AllOutputs>(&outputsSpec))
+        return "^*";
+
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        return "^" + concatStringsSep(",", *outputNames);
+
+    assert(false);
+}
+
+void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
+{
+    if (std::get_if<DefaultOutputs>(&outputsSpec))
+        json = nullptr;
+
+    else if (std::get_if<AllOutputs>(&outputsSpec))
+        json = std::vector<std::string>({"*"});
+
+    else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        json = *outputNames;
+}
+
+void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
+{
+    if (json.is_null())
+        outputsSpec = DefaultOutputs();
+    else {
+        auto names = json.get<OutputNames>();
+        if (names == OutputNames({"*"}))
+            outputsSpec = AllOutputs();
+        else
+            outputsSpec = names;
+    }
+}
+
 }

src/libstore/path-with-outputs.hh

@@ -4,6 +4,7 @@
 
 #include "path.hh"
 #include "derived-path.hh"
+#include "nlohmann/json_fwd.hpp"
 
 namespace nix {
 
@@ -32,4 +33,25 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
+typedef std::set<std::string> OutputNames;
+
+struct AllOutputs {
+    bool operator < (const AllOutputs & _) const { return false; }
+};
+
+struct DefaultOutputs {
+    bool operator < (const DefaultOutputs & _) const { return false; }
+};
+
+typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
+
+/* Parse a string of the form 'prefix^output1,...outputN' or
+   'prefix^*', returning the prefix and the outputs spec. */
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
+
+std::string printOutputsSpec(const OutputsSpec & outputsSpec);
+
+void to_json(nlohmann::json &, const OutputsSpec &);
+void from_json(const nlohmann::json &, OutputsSpec &);
+
 }

src/libstore/tests/path-with-outputs.cc

@@ -0,0 +1,46 @@
+#include "path-with-outputs.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(parseOutputsSpec, basic)
+{
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
+        ASSERT_EQ(prefix, "foo^bar");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
+        ASSERT_EQ(prefix, "foo^&*()");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+}
+
+}

src/libutil/experimental-features.cc

@@ -58,11 +58,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
     return str << showExperimentalFeature(feature);
 }
 
-void to_json(nlohmann::json& j, const ExperimentalFeature& feature) {
+void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
+{
     j = showExperimentalFeature(feature);
 }
 
-void from_json(const nlohmann::json& j, ExperimentalFeature& feature) {
+void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
+{
     const std::string input = j;
     const auto parsed = parseExperimentalFeature(input);
 

src/libutil/experimental-features.hh

@@ -55,7 +55,7 @@ public:
  * Semi-magic conversion to and from json.
  * See the nlohmann/json readme for more details.
  */
-void to_json(nlohmann::json&, const ExperimentalFeature&);
-void from_json(const nlohmann::json&, ExperimentalFeature&);
+void to_json(nlohmann::json &, const ExperimentalFeature &);
+void from_json(const nlohmann::json &, ExperimentalFeature &);
 
 }

src/nix/bundle.cc

@@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
 
         auto val = installable->toValue(*evalState).first;
 
-        auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+        auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
         InstallableFlake bundler{this,
-            evalState, std::move(bundlerFlakeRef), bundlerName,
+            evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
             {"bundlers." + settings.thisSystem.get() + ".default",
              "defaultBundler." + settings.thisSystem.get()
             },

src/nix/develop.cc

@@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
                 state,
                 installable->nixpkgsFlakeRef(),
                 "bashInteractive",
+                DefaultOutputs(),
                 Strings{},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);

src/nix/flake.cc

@@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
         auto installable = InstallableFlake(nullptr,
-            evalState, std::move(templateFlakeRef), templateName,
+            evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
             defaultTemplateAttrPaths,
             defaultTemplateAttrPathsPrefixes,
             lockFlags);