diff --git a/.gitignore b/.gitignore index 8ceff4ef23e..e25fd7d0c19 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ perl/Makefile.config /tests/shell.drv /tests/config.nix /tests/ca/config.nix +/tests/dyn-drv/config.nix /tests/repl-result-out # /tests/lang/ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 510f674eb4c..4f93f3d9872 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1092,7 +1092,7 @@ drvName, Bindings * attrs, Value & v) bool isImpure = false; std::optional outputHash; std::string outputHashAlgo; - std::optional ingestionMethod; + std::optional ingestionMethod; StringSet outputs; outputs.insert("out"); @@ -1105,7 +1105,10 @@ drvName, Bindings * attrs, Value & v) auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else + else if (s == "text") { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + ingestionMethod = TextIngestionMethod {}; + } else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .errPos = state.positions[noPos] @@ -1273,11 +1276,16 @@ drvName, Bindings * attrs, Value & v) })); /* Check whether the derivation name is valid. */ - if (isDerivation(drvName)) + if (isDerivation(drvName) && + !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + outputs.size() == 1 && + *(outputs.begin()) == "out")) + { state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), .errPos = state.positions[noPos] })); + } if (outputHash) { /* Handle fixed-output derivations. @@ -1293,21 +1301,15 @@ drvName, Bindings * attrs, Value & v) auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, - .references = {}, - }); - drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", - DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = method, - .hash = std::move(h), - }, - }); + + DerivationOutput::CAFixed dof { + .ca = ContentAddress::fromParts( + std::move(method), + std::move(h)), + }; + + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + drv.outputs.insert_or_assign("out", std::move(dof)); } else if (contentAddressed || isImpure) { @@ -1325,13 +1327,13 @@ drvName, Bindings * attrs, Value & v) if (isImpure) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { - .method = method, + .method = method.raw, .hashType = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { - .method = method, + .method = method.raw, .hashType = ht, }); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a4bb94b0ee2..af153ebfb4c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation() ) ) ); - else + else { + auto * cap = getDerivationCA(*drv); addWaitee(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + cap ? std::optional { *cap } : std::nullopt))); + } } if (waitees.empty()) /* to prevent hang (no wake-up event) */ diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 21cd6e7ee39..2e4020f338e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2426,37 +2426,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() throw BuildError( "output path %1% without valid stats info", actualPath); - if (outputHash.method == FileIngestionMethod::Flat) { + if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || + outputHash.method == ContentAddressMethod { TextIngestionMethod {} }) + { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } rewriteOutput(); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(actualPath, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + }, + }, outputHash.method.raw); auto got = caSink.finish().first; + + auto optCA = ContentAddressWithReferences::fromPartsOpt( + outputHash.method, + std::move(got), + rewriteRefs()); + if (!optCA) { + // TODO track distinct failure modes separately (at the time of + // writing there is just one but `nullopt` is unclear) so this + // message can't get out of sync. + throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing"); + } ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - FixedOutputInfo { - .hash = { - .method = outputHash.method, - .hash = got, - }, - .references = rewriteRefs(), - }, + *std::move(optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2503,13 +2517,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { + auto wanted = dof.ca.getHash(); + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, + .method = dof.ca.getMethod(), + .hashType = wanted.type, }); /* Check wanted hash */ - const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); auto got = newInfo0.ca->getHash(); if (wanted != got) { @@ -2522,6 +2537,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() wanted.to_string(SRI, true), got.to_string(SRI, true))); } + if (!newInfo0.references.empty()) + delayedException = std::make_exception_ptr( + BuildError("illegal path references in fixed-output derivation '%s'", + worker.store.printStorePath(drvPath))); + return newInfo0; }, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 055b216db5c..04f7ac214f4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) } } +std::string ContentAddressMethod::renderPrefix() const +{ + return std::visit(overloaded { + [](TextIngestionMethod) -> std::string { return "text:"; }, + [](FileIngestionMethod m2) { + /* Not prefixed for back compat with things that couldn't produce text before. */ + return makeFileIngestionPrefix(m2); + }, + }, raw); +} + +ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) +{ + ContentAddressMethod method = FileIngestionMethod::Flat; + if (splitPrefix(m, "r:")) + method = FileIngestionMethod::Recursive; + else if (splitPrefix(m, "text:")) + method = TextIngestionMethod {}; + return method; +} + std::string ContentAddress::render() const { return std::visit(overloaded { @@ -36,14 +57,14 @@ std::string ContentAddress::render() const }, raw); } -std::string ContentAddressMethod::render() const +std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { - [](const TextHashMethod & th) { - return std::string{"text:"} + printHashType(htSHA256); + [&](const TextIngestionMethod & th) { + return std::string{"text:"} + printHashType(ht); }, - [](const FixedOutputHashMethod & fshm) { - return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); + [&](const FileIngestionMethod & fim) { + return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); } }, raw); } @@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const /** * Parses content address strings up to the hash. */ -static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. HashType hashType = parseHashType_(); - if (hashType != htSHA256) - throw Error("text content address hash should use %s, but instead uses %s", - printHashType(htSHA256), printHashType(hashType)); - return TextHashMethod {}; + return { + TextIngestionMethod {}, + std::move(hashType), + }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; HashType hashType = parseHashType_(); - return FixedOutputHashMethod { - .fileIngestionMethod = method, - .hashType = std::move(hashType), + return { + std::move(method), + std::move(hashType), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress ContentAddress::parse(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) +{ auto rest = rawCa; - ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); + auto hashType = hashType_; // work around clang bug - return std::visit( - overloaded { - [&](TextHashMethod & thm) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) - }); - }, - [&](FixedOutputHashMethod & fohMethod) { - return ContentAddress(FixedOutputHash { - .method = fohMethod.fileIngestionMethod, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), - }); - }, - }, caMethod.raw); + return std::visit(overloaded { + [&](TextIngestionMethod &) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, hashType) + }); + }, + [&](FileIngestionMethod & fim) { + return ContentAddress(FixedOutputHash { + .method = fim, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }); + }, + }, caMethod.raw); } -ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } +ContentAddress ContentAddress::fromParts( + ContentAddressMethod method, Hash hash) noexcept +{ + return std::visit(overloaded { + [&](TextIngestionMethod _) -> ContentAddress { + return TextHash { + .hash = std::move(hash), + }; + }, + [&](FileIngestionMethod m2) -> ContentAddress { + return FixedOutputHash { + .method = std::move(m2), + .hash = std::move(hash), + }; + }, + }, method.raw); +} + +ContentAddressMethod ContentAddress::getMethod() const +{ + return std::visit(overloaded { + [](const TextHash & th) -> ContentAddressMethod { + return TextIngestionMethod {}; + }, + [](const FixedOutputHash & fsh) -> ContentAddressMethod { + return fsh.method; + }, + }, raw); +} + const Hash & ContentAddress::getHash() const { return std::visit(overloaded { @@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const }, raw); } +std::string ContentAddress::printMethodAlgo() const +{ + return getMethod().renderPrefix() + + printHashType(getHash().type); +} + bool StoreReferences::empty() const { return !self && others.empty(); @@ -156,7 +214,8 @@ size_t StoreReferences::size() const return (self ? 1 : 0) + others.size(); } -ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept +{ return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { @@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con }, ca.raw); } +std::optional ContentAddressWithReferences::fromPartsOpt( + ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept +{ + return std::visit(overloaded { + [&](TextIngestionMethod _) -> std::optional { + if (refs.self) + return std::nullopt; + return ContentAddressWithReferences { + TextInfo { + .hash = { .hash = std::move(hash) }, + .references = std::move(refs.others), + } + }; + }, + [&](FileIngestionMethod m2) -> std::optional { + return ContentAddressWithReferences { + FixedOutputInfo { + .hash = { + .method = m2, + .hash = std::move(hash), + }, + .references = std::move(refs), + } + }; + }, + }, method.raw); +} + +ContentAddressMethod ContentAddressWithReferences::getMethod() const +{ + return std::visit(overloaded { + [](const TextInfo & th) -> ContentAddressMethod { + return TextIngestionMethod {}; + }, + [](const FixedOutputInfo & fsh) -> ContentAddressMethod { + return fsh.hash.method; + }, + }, raw); +} + +Hash ContentAddressWithReferences::getHash() const +{ + return std::visit(overloaded { + [](const TextInfo & th) { + return th.hash.hash; + }, + [](const FixedOutputInfo & fsh) { + return fsh.hash.hash; + }, + }, raw); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 2f98950fb9a..e1e80448ba0 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -21,8 +21,14 @@ namespace nix { * * Somewhat obscure, used by \ref Derivation derivations and * `builtins.toFile` currently. + * + * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that + * the former may not have self-references and is tagged `text:${algo}:${hash}` + * rather than `fixed:${algo}:${hash}`. The contents of the store path are + * ingested and hashed identically, aside from the slightly different tag and + * restriction on self-references. */ -struct TextHashMethod : std::monostate { }; +struct TextIngestionMethod : std::monostate { }; /** * An enumeration of the main ways we can serialize file system @@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t { */ std::string makeFileIngestionPrefix(FileIngestionMethod m); -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; - - GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); -}; - /** * An enumeration of all the ways we can serialize file system objects. * @@ -64,8 +63,8 @@ struct FixedOutputHashMethod { struct ContentAddressMethod { typedef std::variant< - TextHashMethod, - FixedOutputHashMethod + TextIngestionMethod, + FileIngestionMethod > Raw; Raw raw; @@ -77,9 +76,36 @@ struct ContentAddressMethod : raw(std::forward(arg)...) { } - static ContentAddressMethod parse(std::string_view rawCaMethod); - std::string render() const; + /** + * Parse the prefix tag which indicates how the files + * were ingested, with the fixed output case not prefixed for back + * compat. + * + * @param [in] m A string that should begin with the prefix. + * @param [out] m The remainder of the string after the prefix. + */ + static ContentAddressMethod parsePrefix(std::string_view & m); + + /** + * Render the prefix tag which indicates how the files wre ingested. + * + * The rough inverse of `parsePrefix()`. + */ + std::string renderPrefix() const; + + /** + * Parse a content addressing method and hash type. + */ + static std::pair parse(std::string_view rawCaMethod); + + /** + * Render a content addressing method and hash type in a + * nicer way, prefixing both cases. + * + * The rough inverse of `parse()`. + */ + std::string render(HashType ht) const; }; @@ -147,8 +173,9 @@ struct ContentAddress { } /** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + * Compute the content-addressability assertion + * (`ValidPathInfo::ca`) for paths created by + * `Store::makeFixedOutputPath()` / `Store::addToStore()`. */ std::string render() const; @@ -156,9 +183,27 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); + /** + * Create a `ContentAddress` from 2 parts: + * + * @param method Way ingesting the file system data. + * + * @param hash Hash of ingested file system data. + */ + static ContentAddress fromParts( + ContentAddressMethod method, Hash hash) noexcept; + + ContentAddressMethod getMethod() const; + const Hash & getHash() const; + + std::string printMethodAlgo() const; }; +/** + * Render the `ContentAddress` if it exists to a string, return empty + * string otherwise. + */ std::string renderContentAddress(std::optional ca); @@ -244,10 +289,29 @@ struct ContentAddressWithReferences { } /** - * Create a ContentAddressWithReferences from a mere ContentAddress, by - * assuming no references in all cases. + * Create a `ContentAddressWithReferences` from a mere + * `ContentAddress`, by claiming no references. */ - static ContentAddressWithReferences withoutRefs(const ContentAddress &); + static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept; + + /** + * Create a `ContentAddressWithReferences` from 3 parts: + * + * @param method Way ingesting the file system data. + * + * @param hash Hash of ingested file system data. + * + * @param refs References to other store objects or oneself. + * + * Do note that not all combinations are supported; `nullopt` is + * returns for invalid combinations. + */ + static std::optional fromPartsOpt( + ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; + + ContentAddressMethod getMethod() const; + + Hash getHash() const; }; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index af9a76f1e70..5083497a97e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); + auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); + auto hashType = hashType_; // work around clang bug FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](const TextHashMethod &) { + [&](const TextIngestionMethod &) { + if (hashType != htSHA256) + throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", + name, printHashType(hashType)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](const FixedOutputHashMethod & fohm) { - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); + [&](const FileIngestionMethod & fim) { + auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod.raw); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 15f3908ed5f..d56dc727bcd 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" +#include "split.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" #include @@ -35,9 +36,9 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { - return store.makeFixedOutputPath( + return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), - { hash, {} }); + ContentAddressWithReferences::withoutRefs(ca)); } @@ -211,29 +212,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hash) + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) { if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (hashAlgo.substr(0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (method == TextIngestionMethod {}) + experimentalFeatureSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); - if (hash == "impure") { + if (hashS == "impure") { experimentalFeatureSettings.require(Xp::ImpureDerivations); assert(pathS == ""); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), }; - } else if (hash != "") { + } else if (hashS != "") { validatePath(pathS); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, + .ca = ContentAddress::fromParts( + std::move(method), + std::move(hash)), }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -393,12 +392,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, }, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); - s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -409,7 +408,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutputImpure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw()); @@ -626,8 +625,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" - + dof.hash.printMethodAlgo() + ":" - + dof.hash.hash.to_string(Base16, false) + ":" + + dof.ca.printMethodAlgo() + ":" + + dof.ca.getHash().to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -777,12 +776,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) - << dof.hash.printMethodAlgo() - << dof.hash.hash.to_string(Base16, false); + << dof.ca.printMethodAlgo() + << dof.ca.getHash().to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashType(dof.hashType)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -792,7 +791,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::Impure & doi) { out << "" - << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashType(doi.hashType)) << "impure"; }, }, i.second.raw()); @@ -942,7 +941,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const envHasRightPath(doia.path, i.first); }, [&](const DerivationOutput::CAFixed & dof) { - StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} }); + auto path = dof.path(store, drvName, i.first); envHasRightPath(path, i.first); }, [&](const DerivationOutput::CAFloating &) { @@ -971,15 +970,16 @@ nlohmann::json DerivationOutput::toJSON( }, [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); - res["hashAlgo"] = dof.hash.printMethodAlgo(); - res["hash"] = dof.hash.hash.to_string(Base16, false); + res["hashAlgo"] = dof.ca.printMethodAlgo(); + res["hash"] = dof.ca.getHash().to_string(Base16, false); + // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); + res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); + res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); res["impure"] = true; }, }, raw()); @@ -998,15 +998,15 @@ DerivationOutput DerivationOutput::fromJSON( for (const auto & [key, _] : json) keys.insert(key); - auto methodAlgo = [&]() -> std::pair { + auto methodAlgo = [&]() -> std::pair { std::string hashAlgo = json["hashAlgo"]; - auto method = FileIngestionMethod::Flat; - if (hashAlgo.substr(0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } - auto hashType = parseHashType(hashAlgo); - return { method, hashType }; + // remaining to parse, will be mutated by parsers + std::string_view s = hashAlgo; + ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); + if (method == TextIngestionMethod {}) + xpSettings.require(Xp::DynamicDerivations); + auto hashType = parseHashType(s); + return { std::move(method), std::move(hashType) }; }; if (keys == (std::set { "path" })) { @@ -1018,10 +1018,9 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .hash = { - .method = method, - .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), - }, + .ca = ContentAddress::fromParts( + std::move(method), + Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); @@ -1032,8 +1031,8 @@ DerivationOutput DerivationOutput::fromJSON( xpSettings.require(Xp::CaDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::CAFloating { - .method = method, - .hashType = hashType, + .method = std::move(method), + .hashType = std::move(hashType), }; } @@ -1045,7 +1044,7 @@ DerivationOutput DerivationOutput::fromJSON( xpSettings.require(Xp::ImpureDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::Impure { - .method = method, + .method = std::move(method), .hashType = hashType, }; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d00b23b6dfc..1e2143f3123 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -36,9 +36,11 @@ struct DerivationOutputInputAddressed struct DerivationOutputCAFixed { /** - * hash used for expected hash computation + * Method and hash used for expected hash computation. + * + * References are not allowed by fiat. */ - FixedOutputHash hash; + ContentAddress ca; /** * Return the \ref StorePath "store path" corresponding to this output @@ -48,7 +50,7 @@ struct DerivationOutputCAFixed */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; - GENERATE_CMP(DerivationOutputCAFixed, me->hash); + GENERATE_CMP(DerivationOutputCAFixed, me->ca); }; /** @@ -61,7 +63,7 @@ struct DerivationOutputCAFloating /** * How the file system objects will be serialized for hashing */ - FileIngestionMethod method; + ContentAddressMethod method; /** * How the serialization will be hashed @@ -88,7 +90,7 @@ struct DerivationOutputImpure /** * How the file system objects will be serialized for hashing */ - FileIngestionMethod method; + ContentAddressMethod method; /** * How the serialization will be hashed @@ -343,12 +345,14 @@ struct Derivation : BasicDerivation Store & store, const std::map, StorePath> & inputDrvOutputs) const; - /* Check that the derivation is valid and does not present any - illegal states. - - This is mainly a matter of checking the outputs, where our C++ - representation supports all sorts of combinations we do not yet - allow. */ + /** + * Check that the derivation is valid and does not present any + * illegal states. + * + * This is mainly a matter of checking the outputs, where our C++ + * representation supports all sorts of combinations we do not yet + * allow. + */ void checkInvariants(Store & store, const StorePath & drvPath) const; Derivation() = default; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 89148d4152c..50336c77971 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +const ContentAddress * getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) { - if (const auto * v = std::get_if(&out->second.raw())) - return v->hash; + if (out == drv.outputs.end()) + return nullptr; + if (auto dof = std::get_if(&out->second)) { + return &dof->ca; } - return std::nullopt; + return nullptr; } void Store::queryMissing(const std::vector & targets, @@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector & targets, if (drvState_->lock()->done) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); + auto * cap = getDerivationCA(*drv); + querySubstitutablePathInfos({ + { + outPath, + cap ? std::optional { *cap } : std::nullopt, + }, + }, infos); if (infos.empty()) { drvState_->lock()->done = true; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a6e8b957737..0ed17a6ce50 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -597,6 +597,7 @@ ref RemoteStore::addCAToStore( Source & dump, std::string_view name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair) { @@ -608,7 +609,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << caMethod.render(); + << caMethod.render(hashType); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -628,26 +629,29 @@ ref RemoteStore::addCAToStore( if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); std::visit(overloaded { - [&](const TextHashMethod & thm) -> void { + [&](const TextIngestionMethod & thm) -> void { + if (hashType != htSHA256) + throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", + name, printHashType(hashType)); std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; worker_proto::write(*this, conn->to, references); conn.processStderr(); }, - [&](const FixedOutputHashMethod & fohm) -> void { + [&](const FileIngestionMethod & fim) -> void { conn->to << wopAddToStore << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); + << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashType); try { conn->to.written = 0; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::Recursive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -678,7 +682,7 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; + return addCAToStore(dump, name, method, hashType, references, repair)->path; } @@ -778,7 +782,7 @@ StorePath RemoteStore::addTextToStore( RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a30466647ad..82e4656ab96 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -78,6 +78,7 @@ public: Source & dump, std::string_view name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c910d1c96ea..bad61001405 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1022,7 +1022,7 @@ std::optional decodeValidPathInfo( */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6f94904ddd8..6328ad3700e 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest } }; +class DynDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + } +}; + class ImpureDerivationTest : public DerivationTest { void SetUp() override @@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationTest, caFixed, +TEST_JSON(DerivationTest, caFixedFlat, + R"({ + "hashAlgo": "sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedNAR, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", (DerivationOutput::CAFixed { - .hash = { + .ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), "drv-name", "output-name") +TEST_JSON(DynDerivationTest, caFixedText, + R"({ + "hashAlgo": "text:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = TextHash { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + TEST_JSON(CaDerivationTest, caFloating, R"({ "hashAlgo": "r:sha256" diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index bd189966253..ad0ec0427e5 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -199,6 +199,16 @@ constexpr std::array xpFeatureDetails = {{ networking. )", }, + { + .tag = Xp::DynamicDerivations, + .name = "dynamic-derivations", + .description = R"( + Allow the use of a few things related to dynamic derivations: + + - "text hashing" derivation outputs, so we can build .drv + files. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 3c00bc4e55b..409100592a8 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -29,6 +29,7 @@ enum struct ExperimentalFeature Cgroups, DiscardReferences, DaemonTrustOverride, + DynamicDerivations, }; /** diff --git a/tests/dyn-drv/common.sh b/tests/dyn-drv/common.sh new file mode 100644 index 00000000000..c786f6925fb --- /dev/null +++ b/tests/dyn-drv/common.sh @@ -0,0 +1,8 @@ +source ../common.sh + +# Need backend to support text-hashing too +requireDaemonNewerThan "2.16.0pre20230419" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon diff --git a/tests/dyn-drv/config.nix.in b/tests/dyn-drv/config.nix.in new file mode 120000 index 00000000000..af24ddb30b0 --- /dev/null +++ b/tests/dyn-drv/config.nix.in @@ -0,0 +1 @@ +../config.nix.in \ No newline at end of file diff --git a/tests/dyn-drv/recursive-mod-json.nix b/tests/dyn-drv/recursive-mod-json.nix new file mode 100644 index 00000000000..c6a24ca4f3b --- /dev/null +++ b/tests/dyn-drv/recursive-mod-json.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +let innerName = "foo"; in + +mkDerivation rec { + name = "${innerName}.drv"; + SHELL = shell; + + requiredSystemFeatures = [ "recursive-nix" ]; + + drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath; + + buildCommand = '' + export NIX_CONFIG='experimental-features = nix-command ca-derivations' + + PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH + + # JSON of pre-existing drv + nix derivation show $drv | jq .[] > drv0.json + + # Fix name + jq < drv0.json '.name = "${innerName}"' > drv1.json + + # Extend `buildCommand` + jq < drv1.json '.env.buildCommand += "echo \"I am alive!\" >> $out/hello\n"' > drv0.json + + # Used as our output + cp $(nix derivation add < drv0.json) $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; +} diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/dyn-drv/recursive-mod-json.sh new file mode 100644 index 00000000000..070c5c2cb80 --- /dev/null +++ b/tests/dyn-drv/recursive-mod-json.sh @@ -0,0 +1,25 @@ +source common.sh + +# FIXME +if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi + +enableFeatures 'recursive-nix' +restartDaemon + +clearStore + +rm -f $TEST_ROOT/result + +EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq)) +export EXTRA_PATH + +# Will produce a drv +metaDrv=$(nix-instantiate ./recursive-mod-json.nix) + +# computed "dynamic" derivation +drv=$(nix-store -r $metaDrv) + +# build that dyn drv +res=$(nix-store -r $drv) + +grep 'I am alive!' $res/hello diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/dyn-drv/text-hashed-output.nix new file mode 100644 index 00000000000..a700fd102de --- /dev/null +++ b/tests/dyn-drv/text-hashed-output.nix @@ -0,0 +1,29 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + hello = mkDerivation { + name = "hello"; + buildCommand = '' + set -x + echo "Building a CA derivation" + mkdir -p $out + echo "Hello World" > $out/hello + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + producingDrv = mkDerivation { + name = "hello.drv"; + buildCommand = '' + echo "Copying the derivation" + cp ${builtins.unsafeDiscardOutputDependency hello.drvPath} $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }; +} diff --git a/tests/dyn-drv/text-hashed-output.sh b/tests/dyn-drv/text-hashed-output.sh new file mode 100644 index 00000000000..f3e5aa93bb7 --- /dev/null +++ b/tests/dyn-drv/text-hashed-output.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named root, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of root, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate the root derivation +# - build the dependent derivation +# - check that the path of the output coincides with that of the original derivation + +drv=$(nix-instantiate ./text-hashed-output.nix -A hello) +nix show-derivation "$drv" + +drvProducingDrv=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) +nix show-derivation "$drvProducingDrv" + +out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) + +nix path-info $drv --derivation --json | jq +nix path-info $out1 --derivation --json | jq + +test $out1 == $drv diff --git a/tests/local.mk b/tests/local.mk index 7c3b425993c..d37c21abf5d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -110,6 +110,8 @@ nix_tests = \ ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ + dyn-drv/text-hashed-output.sh \ + dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -137,11 +139,19 @@ ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif -install-tests += $(foreach x, $(nix_tests), tests/$(x)) +install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -clean-files += $(d)/common/vars-and-functions.sh $(d)/config.nix $(d)/ca/config.nix +clean-files += \ + $(d)/common/vars-and-functions.sh \ + $(d)/config.nix \ + $(d)/ca/config.nix \ + $(d)/dyn-drv/config.nix -test-deps += tests/common/vars-and-functions.sh tests/config.nix tests/ca/config.nix +test-deps += \ + tests/common/vars-and-functions.sh \ + tests/config.nix \ + tests/ca/config.nix \ + tests/dyn-drv/config.nix ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) diff --git a/tests/recursive.sh b/tests/recursive.sh index 6335d44a526..27e1674ca9e 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -1,11 +1,11 @@ source common.sh -sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf -restartDaemon - # FIXME if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi +enableFeatures 'recursive-nix' +restartDaemon + clearStore rm -f $TEST_ROOT/result