From a2f0ba6a6dbb79efbb83ebe92b287f79b3f3af91 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 16:51:49 -0400 Subject: [PATCH 01/29] Fix transitive input locking. Fixes reproducibility issue described in #9143 Fixes #9143 --- src/libexpr/flake/flake.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f4b..5d9d60655b4 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -351,10 +351,13 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); std::map overrides; + std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; - for (auto & i : lockFlags.inputOverrides) + for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + explicitCliOverrides.insert(i.first); + } LockFile newLockFile; @@ -425,6 +428,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end(); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we @@ -460,7 +464,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref - && !hasOverride) + && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -541,7 +545,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = input2.ref ? *input2.ref : *input.ref; + auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; From 311e2ad024441950cb1300e56c9745259deebdda Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Wed, 18 Oct 2023 10:37:06 -0400 Subject: [PATCH 02/29] Address review comments --- src/libexpr/flake/flake.cc | 2 +- tests/functional/flakes/follow-paths.sh | 76 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5d9d60655b4..2c7e12ec9ed 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -428,7 +428,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); - bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(inputPath); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 8573b55113d..7f4e8bf5df0 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -260,3 +260,79 @@ EOF checkRes=$(nix flake lock "$flakeFollowCycle" 2>&1 && fail "nix flake lock should have failed." || true) echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" + + +# Test transitive input url locking +# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143 +# +# We construct the following graph, where p->q means p has input q. +# +# A -> B -> C +# +# And override B/C to flake D, first in A's flake.nix and then with --override-input. +# +# A -> B -> D +flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA" +flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB" +flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC" +flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD" + + +createGitRepo "$flakeFollowsCustomUrlA" +mkdir -p "$flakeFollowsCustomUrlB" +mkdir -p "$flakeFollowsCustomUrlC" +mkdir -p "$flakeFollowsCustomUrlD" + +cat > "$flakeFollowsCustomUrlD/flake.nix" < "$flakeFollowsCustomUrlC/flake.nix" < "$flakeFollowsCustomUrlB/flake.nix" < "$flakeFollowsCustomUrlA/flake.nix" < Date: Mon, 20 Nov 2023 03:37:02 -0700 Subject: [PATCH 03/29] flakes: check for flake.nix before complaining that lstat on it fails getFlake currently calls lstat (via isLink via canonPath) before it performs the sanity check that a flake.nix exists in the first place. This commit moves the check to before path canonicalization, so that failed symlink check operations don't throw before the check does. --- src/libexpr/flake/flake.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 54de53e0bfd..b128de31e1e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -212,8 +212,16 @@ static Flake getFlake( auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); + // We need to guard against symlink attacks, but before we start doing + // filesystem operations we should make sure there's a flake.nix in the + // first place. + auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir; + auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix"; + if (!pathExists(unsafeFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + // Guard against symlink attacks. - auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(unsafeFlakeDir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", @@ -226,9 +234,6 @@ static Flake getFlake( .storePath = storePath, }; - if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); - Value vInfo; state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack From f880469173061a07f0b2a24734932c5a9ad633c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 19 Nov 2023 10:17:57 -0500 Subject: [PATCH 04/29] Put `canonicaliseTimestampAndPermissions` in its own header/file It is not inherently tied to `LocalStore`, it could probably even go in `libnixutil`. Functions not attached to `LocalStore` should not be declared in `local-store.hh`. I am moving it to facilitate experimenting for #9344. If canonicalisation should be done client-side in client-side builds, there wouldn't be a `LocalStore` at all so having to include that header to get this freestanding function is cumbersome and wrong. Perhaps canonicalisation should still be done server-side for security reasons --- I don't mean to make that judgement call now --- but even if so, this freestanding function still isn't connected to `LocalStore` so while less urgent it is still better to move out of this header. --- src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/local-store.cc | 159 +----------------- src/libstore/local-store.hh | 34 ---- src/libstore/optimise-store.cc | 1 + src/libstore/posix-fs-canonicalise.cc | 169 ++++++++++++++++++++ src/libstore/posix-fs-canonicalise.hh | 45 ++++++ src/nix-store/nix-store.cc | 1 + 7 files changed, 218 insertions(+), 192 deletions(-) create mode 100644 src/libstore/posix-fs-canonicalise.cc create mode 100644 src/libstore/posix-fs-canonicalise.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a9f9307737a..198402ff7c8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -19,6 +19,7 @@ #include "namespaces.hh" #include "child.hh" #include "unix-domain-socket.hh" +#include "posix-fs-canonicalise.hh" #include #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a3582ad8ea..4ff75f528c3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -11,6 +11,7 @@ #include "finally.hh" #include "compression.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -581,164 +582,6 @@ void LocalStore::makeStoreWritable() } -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - canonicaliseTimestampAndPermissions(path, lstat(path)); -} - - -static void canonicalisePathMetaData_( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - auto st = lstat(path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); - } -} - - -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, uidRange, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); -} - - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d589bee5b1..8f0ffd2a2be 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -371,38 +371,4 @@ private: friend struct DerivationGoal; }; - -typedef std::pair Inode; -typedef std::set InodesSeen; - - -/** - * "Fix", or canonicalise, the meta-data of the files in a store path - * after it has been built. In particular: - * - * - the last modification date on each file is set to 1 (i.e., - * 00:00:01 1/1/1970 UTC) - * - * - the permissions are set of 444 or 555 (i.e., read-only with or - * without execute permission; setuid bits etc. are cleared) - * - * - the owner and group are set to the Nix user and group, if we're - * running as root. - * - * If uidRange is not empty, this function will throw an error if it - * encounters files owned by a user outside of the closed interval - * [uidRange->first, uidRange->second]. - */ -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a4ac413b343..0fa977545c7 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "local-store.hh" #include "globals.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc new file mode 100644 index 00000000000..cc3ab0b74c3 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.cc @@ -0,0 +1,169 @@ +#include + +#include "posix-fs-canonicalise.hh" +#include "file-system.hh" +#include "signals.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +namespace nix { + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError("changing modification time of '%1%'", path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + canonicaliseTimestampAndPermissions(path, lstat(path)); +} + + +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + + auto st = lstat(path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error("file '%1%' has an unsupported type", path); + +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) + throw BuildError("invalid ownership on file '%1%'", path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto & i : entries) + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); + } +} + + +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, uidRange, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + auto st = lstat(path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error("wrong ownership of top-level store path '%1%'", path); + } +} + + +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, uidRange, inodesSeen); +} + +} diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh new file mode 100644 index 00000000000..35644af125f --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" +#include "error.hh" + +namespace nix { + +typedef std::pair Inode; +typedef std::set InodesSeen; + + +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 123283dfeea..25f0107bc4f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -14,6 +14,7 @@ #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "posix-fs-canonicalise.hh" #include #include From 2ce8c9650b3e714f28d8685e48996141cba2df2c Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 23 Nov 2023 22:02:20 +0100 Subject: [PATCH 05/29] doc: primops: add more info for foldl (#9254) * doc: primops: add more info for foldl From the existing doc it is not obvious whether the first or the second argument is the accumulator. This is however relevant to know, as for certain scenarios, this might change the behavior. Co-authored-by: Valentin Gagarin --- src/libexpr/primops.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a8d44d8b7a4..7c056141372 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3179,9 +3179,16 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. - The return value of each application of `op` is evaluated immediately, - even for intermediate values. + ...`. + + For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates + to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} + ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. + + The first argument of `op` is the accumulator wheres the second + argument is the current element being processed. The return value + of each application of `op` is evaluated immediately, even for + intermediate values. )", .fun = prim_foldlStrict, }); From 5be0e6b314c216b0b51499fc488ca08272297469 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 24 Nov 2023 10:50:01 +0100 Subject: [PATCH 06/29] doc: primops: fix typo --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c056141372..ba735b4354a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3185,7 +3185,7 @@ static RegisterPrimOp primop_foldlStrict({ to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. - The first argument of `op` is the accumulator wheres the second + The first argument of `op` is the accumulator whereas the second argument is the current element being processed. The return value of each application of `op` is evaluated immediately, even for intermediate values. From 6a94755b1240be654cadb463a9f528eeccf3787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 24 Nov 2023 11:45:37 +0100 Subject: [PATCH 07/29] Allow user input in `git commit` We occasionnally commit to git repositories (like with `nix flake update --commit-lock-file`). This shells out to `git commit`, which might wait for user input (for a signing key passphrase for instance). Disable the progress bar while this is running to make sure that the user can enter it. --- src/libfetchers/git.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2c5b70f53eb..8cd74057ce0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -11,6 +11,8 @@ #include "fs-input-accessor.hh" #include "mounted-input-accessor.hh" #include "git-utils.hh" +#include "logging.hh" +#include "finally.hh" #include "fetch-settings.hh" @@ -314,6 +316,9 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); if (commitMsg) runProgram("git", true, { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); From b7982372d234b1fd15bab01d09093471c1870bb4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 15:05:21 +0100 Subject: [PATCH 08/29] Compile hand-written release notes with changelog-d --- .gitignore | 1 + doc/manual/local.mk | 5 +++- doc/manual/rl-next/config | 2 ++ doc/manual/src/contributing/hacking.md | 38 ++++++++++++++++++++++++++ flake.nix | 11 +++++++- 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 doc/manual/rl-next/config diff --git a/.gitignore b/.gitignore index 04d96ca2c09..38e7e2b5a0a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ perl/Makefile.config /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md /doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh diff --git a/doc/manual/local.mk b/doc/manual/local.mk index db3daf2524a..74a4103b30b 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -144,6 +144,9 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ +$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* + $(trace-gen) changelog-d doc/manual/rl-next > $@ + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html @@ -177,7 +180,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 00000000000..b3c2e868f96 --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index fe08ceb94e9..855900d7a8b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +synopsis: Basically a title +issues: #1234 +prs: #1238 +description: { + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ + +} +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. +Non-releases build the release notes on the fly. diff --git a/flake.nix b/flake.nix index 9030a74f74d..ca37948a7cb 100644 --- a/flake.nix +++ b/flake.nix @@ -173,6 +173,10 @@ "--enable-internal-api-docs" ]; + # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - + # vastly improves output closure, and adds shell completions + changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + nativeBuildDeps = [ buildPackages.bison @@ -190,7 +194,10 @@ buildPackages.jq # Also for custom mdBook preprocessor. buildPackages.openssh # only needed for tests (ssh-keygen) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease) changelog-d + ; buildDeps = [ curl @@ -727,6 +734,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools + # We want changelog-d in the shell even if it's an official release + ++ lib.optional officialRelease changelog-d ; buildInputs = buildDeps ++ propagatedDeps From b26038c517ed10feae751ad6733244c00b715d34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:08:34 +0100 Subject: [PATCH 09/29] doc: Rename 2X.XX to "Upcoming release", and only generate if applicable --- .gitignore | 1 + doc/manual/local.mk | 13 +++++++++++-- doc/manual/src/SUMMARY.md.in | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 38e7e2b5a0a..5c359d7a0c5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ perl/Makefile.config /doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 74a4103b30b..265a4649dd0 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -144,9 +144,18 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ -$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* $(trace-gen) changelog-d doc/manual/rl-next > $@ +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8dc464abd8f..8e7b4eeabb2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -114,7 +114,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +{{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) From 9aa63f70d7d861ba74764188410c0add730cd48d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 24 Nov 2023 15:32:02 +0100 Subject: [PATCH 10/29] fricklerhandwerk: subscribe to documentation changes (#9422) * fricklerhandwerk: subscribe to documentation changes --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ab59086491b..39d595199d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,5 +14,12 @@ /doc @fricklerhandwerk *.md @fricklerhandwerk +# Documentation of built-in functions +src/libexpr/primops.cc @fricklerhandwerk @roberth +# Documentation on experimental features +src/libutil/experimental-features.cc @fricklerhandwerk +# Documentation on configuration settings +src/libstore/globals.hh @fricklerhandwerk + # Libstore layer /src/libstore @thufschmitt From 2a538c571b13877fa426f2cff2749cf17d140216 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:49:11 +0100 Subject: [PATCH 11/29] Add scripts/release-notes --- maintainers/release-process.md | 35 +++----- scripts/release-notes | 148 +++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 22 deletions(-) create mode 100755 scripts/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index d85266b8161..e542d8be52b 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,34 +24,23 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. -* Move the contents of `doc/manual/src/release-notes/rl-next.md` - (except the first line) to - `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is - the contents of `.version` *without* the patch level, e.g. `2.12` - rather than `2.12.0`). +* Compile the release notes by running -* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like - - ``` - # Release 2.12 (2022-12-06) + ```console + $ git checkout -b release-notes + $ VERSION=X.YY ./scripts/release-notes ``` -* Proof-read / edit / rearrange the release notes. Breaking changes - and highlights should go to the top. + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. -* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` - (*not* `SUMMARY.md`), e.g. + A commit is created. - ``` - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - ``` +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. -* Run +* Push. ```console - $ git checkout -b release-notes - $ git add doc/manual/src/release-notes/rl-$VERSION.md - $ git commit -a -m 'Release notes' $ git push --set-upstream $REMOTE release-notes ``` @@ -67,15 +56,17 @@ release: $ git checkout -b $VERSION-maintenance ``` -* Mark the release as stable: +* Mark the release as official: ```console - $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix ``` This removes the link to `rl-next.md` from the manual and sets `officialRelease = true` in `flake.nix`. +* Commit + * Push the release branch: ```console diff --git a/scripts/release-notes b/scripts/release-notes new file mode 100755 index 00000000000..e5ee39d11b7 --- /dev/null +++ b/scripts/release-notes @@ -0,0 +1,148 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small +# ^^^^^^^ +# Only used for bash. shell.nix goes to the flake. + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +case "$VERSION" in + # FIXME: accepts "." without any real digits + [[:digit:]]*.[[:digit:]]*) + ;; + *) + die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" + ;; +esac + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename +title="Release $VERSION ($DATE)" + +( + # TODO add minor number, and append? + echo "# $title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' +) > $file + +log "Wrote $file" + +NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + +# find the marker line, insert new link after it +escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $VERSION" + +report_done From b1ea30f21d24df9afc4eb1635eee9a080e4f81f3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:13:59 +0100 Subject: [PATCH 12/29] scripts/release-notes: Support patch releases This also fixes the broken case statement, which has globs, not regexes. --- maintainers/release-process.md | 24 +++++++++++++ scripts/release-notes | 65 ++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index e542d8be52b..b1259a9bd35 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -150,6 +150,30 @@ release: ## Creating a point release +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./scripts/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + * Wait for the desired evaluation of the maintenance jobset to finish building. diff --git a/scripts/release-notes b/scripts/release-notes index e5ee39d11b7..43f20a547db 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,14 +89,40 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -case "$VERSION" in - # FIXME: accepts "." without any real digits - [[:digit:]]*.[[:digit:]]*) - ;; - *) - die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" - ;; -esac +# mutate/initialize: +# VERSION: MAJOR.MINOR +# FULL_VERSION: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + FULL_VERSION="$VERSION.0" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +log "VERSION=$VERSION" +log "FULL_VERSION=$FULL_VERSION" +log "IS_PATCH=$IS_PATCH" + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi # --- DEFAULTS --- @@ -106,7 +132,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + [0-9]*-[0-9]*-[0-9]*) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" @@ -115,25 +141,28 @@ esac # --- DO THE WORK --- -basename=rl-$VERSION.md -file=doc/manual/src/release-notes/$basename +# menu title="Release $VERSION ($DATE)" +# section on page +section_title="Release $FULL_VERSION ($DATE)" ( # TODO add minor number, and append? - echo "# $title" + echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' -) > $file +) | tee -a $file log "Wrote $file" -NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" -# find the marker line, insert new link after it -escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi for f in doc/manual/rl-next/*.md; do if [[ config != "$(basename $f)" ]]; then From 7c4ee5c8135fae65602791f0b89d0dbae7e94f3e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:36:50 +0100 Subject: [PATCH 13/29] scripts/release-notes: Avoid mutating variables --- scripts/release-notes | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/release-notes b/scripts/release-notes index 43f20a547db..2e1be64a4f2 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,33 +89,35 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -# mutate/initialize: -# VERSION: MAJOR.MINOR -# FULL_VERSION: MAJOR.MINOR.PATCH -# IS_PATCH: true if this is a patch release; append instead of create +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is minor' IS_PATCH=false - FULL_VERSION="$VERSION.0" + version_full="$VERSION.0" + version_major_minor="$VERSION" elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then log 'is minor (.0)' IS_PATCH=false - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is patch' IS_PATCH=true - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" else die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" fi -log "VERSION=$VERSION" -log "FULL_VERSION=$FULL_VERSION" +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" log "IS_PATCH=$IS_PATCH" -basename=rl-$VERSION.md +basename=rl-${version_major_minor}.md file=doc/manual/src/release-notes/$basename if ! $IS_PATCH; then @@ -142,9 +144,9 @@ esac # --- DO THE WORK --- # menu -title="Release $VERSION ($DATE)" +title="Release $version_major_minor ($DATE)" # section on page -section_title="Release $FULL_VERSION ($DATE)" +section_title="Release $version_full ($DATE)" ( # TODO add minor number, and append? @@ -172,6 +174,6 @@ done logcmd git add $file doc/manual/src/SUMMARY.md.in logcmd git status -logcmd git commit -m "release notes: $VERSION" +logcmd git commit -m "release notes: $version_full" report_done From 6971c4adc06d574cbe1e9ab6da19814e11e2ba6c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 19:08:42 +0100 Subject: [PATCH 14/29] maintainers/release-notes <- scripts/release-notes --- {scripts => maintainers}/release-notes | 0 maintainers/release-process.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => maintainers}/release-notes (100%) diff --git a/scripts/release-notes b/maintainers/release-notes similarity index 100% rename from scripts/release-notes rename to maintainers/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index b1259a9bd35..db8b064a514 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -28,7 +28,7 @@ release: ```console $ git checkout -b release-notes - $ VERSION=X.YY ./scripts/release-notes + $ VERSION=X.YY ./maintainers/release-notes ``` where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. @@ -165,7 +165,7 @@ release: * Update release notes. ```console - $ ./scripts/release-notes + $ ./maintainers/release-notes ``` * Push. From 857f9168f7b48aa491052f24fb571c21398f9826 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:27:54 +0100 Subject: [PATCH 15/29] Migrate rl-next.md to doc/manual/rl-next directory --- doc/manual/rl-next/mounted-ssh-store.md | 9 +++++++++ doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++++++++ doc/manual/src/release-notes/rl-next.md | 6 ------ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/mounted-ssh-store.md create mode 100644 doc/manual/rl-next/nix-env-json-drv-path.md delete mode 100644 doc/manual/src/release-notes/rl-next.md diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md new file mode 100644 index 00000000000..39fac528318 --- /dev/null +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -0,0 +1,9 @@ +synopsis: Mounted SSH Store +issues: #7890 +prs: #7912 +description: { + +Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). +This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md new file mode 100644 index 00000000000..fbe2b67d888 --- /dev/null +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -0,0 +1,9 @@ +synopsis: Fix `nix-env --query --drv-path --json` +prs: #9257 +description: { + +Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +} + + diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 0e3d8b46291..00000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release X.Y (202?-??-??) - -- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. - -- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. From 54b684765519dee1e74dca71605f8e7f6c8b0a25 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Fri, 24 Nov 2023 11:17:35 -0500 Subject: [PATCH 16/29] doc: fix machine-specific capabilities leaking --- src/libstore/globals.hh | 14 +++++++++++--- src/libstore/store-api.hh | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 27caf42c4ed..838d2aba29d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -183,7 +183,9 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}, false}; + {"build-cores"}, + // Don't document the machine-specific default value + false}; /** * Read-only mode. Don't copy stuff to the store, don't change @@ -699,7 +701,10 @@ public: Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting systemFeatures{ this, @@ -744,7 +749,10 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting substituters{ this, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 32ad2aa448b..8b6bf9aed33 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -167,7 +167,10 @@ struct StoreConfig : public Config Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. Example: `"kvm"` - )" }; + )", + {}, + // Don't document the machine-specific default value + false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From d2f5e263e3c095dfe9d874387665b88c4bfff6f1 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 6 Feb 2023 16:36:57 +0100 Subject: [PATCH 17/29] Switch from std::regex to boost::regex --- flake.nix | 24 +++++++++++++++++------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 +++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 9030a74f74d..1f7e7b2af43 100644 --- a/flake.nix +++ b/flake.nix @@ -154,7 +154,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" + "--with-boost=${boost-nix}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -202,7 +202,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost + boost-nix lowdown-nix libsodium ] @@ -423,14 +423,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost ]; + disallowedReferences = [ boost-nix ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -440,9 +440,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -470,9 +470,13 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ + -change ${boost-nix}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib + install_name_tool \ + -change ${boost-nix}/lib/libboost_regex.dylib \ + $out/lib/libboost_regex.dylib \ + $out/lib/libnixexpr.dylib ''} ''; @@ -495,6 +499,12 @@ meta.mainProgram = "nix"; }); + boost-nix = final.boost.override { + # enableIcu arg is not yet supported + # but will be with next nixpkgs update + enableIcu = false; + }; + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf94904c..946059339d4 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ba735b4354a..0c34cb6e889 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,6 +16,7 @@ #include "primops.hh" #include +#include #include #include @@ -24,7 +25,6 @@ #include #include -#include #include #include @@ -3885,19 +3885,30 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); +// regex aliases, switch between boost and std +using regex = boost::regex; +using regex_error = boost::regex_error; +using cmatch = boost::cmatch; +using cregex_iterator = boost::cregex_iterator; +namespace regex_constants = boost::regex_constants; +// overloaded function alias +constexpr auto regex_match = [] (auto &&...args) { + return boost::regex_match(std::forward(args)...); + }; + struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - std::regex get(std::string_view re) + regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; } }; @@ -3917,8 +3928,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - std::cmatch match; - if (!std::regex_match(str.begin(), str.end(), match, regex)) { + cmatch match; + if (!regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3933,8 +3944,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -3997,8 +4008,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = std::cregex_iterator(str.begin(), str.end(), regex); - auto end = std::cregex_iterator(); + auto begin = cregex_iterator(str.begin(), str.end(), regex); + auto end = cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4037,8 +4048,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From 213594721ace2b2e2dcf3d57615178a2c1690aa4 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:21:49 +0100 Subject: [PATCH 18/29] gitignore: Also ignore .DS_Store This is a file that Finder on Mac OS loves to add into various folders. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 04d96ca2c09..f7dcea4221f 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,6 @@ result # clangd and possibly more .cache/ + +# Mac OS +.DS_Store From f25c06d7a3289343887d09761c82356a3b6b441b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 26 Nov 2023 15:46:09 +0100 Subject: [PATCH 19/29] docs: Fix broken link Link target definitions need to be in a separate paragraph to be collected. Fixup for https://github.com/NixOS/nix/commit/217d863f7a251a4d8a08ff3294944b45146c61c9 --- doc/manual/src/language/values.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 0bb6567463b..aea68a44170 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -116,6 +116,7 @@ [store path]: ../glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. From d63f72197cec6ff95d9ffc83aa8076acd86a3fd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:48:41 +0100 Subject: [PATCH 20/29] Don't run changelog-d in the build This way we lose the preview of release notes on master, as well as on https://nixos.org/manual/nix/unstable/release-notes/rl-next but we can come back to this. --- doc/manual/local.mk | 8 +++++++- doc/manual/src/contributing/hacking.md | 2 +- flake.nix | 9 ++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 265a4649dd0..f22dfa69e82 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -146,7 +146,13 @@ $(d)/language.json: $(bindir)/nix # Generate "Upcoming release" notes (or clear it and remove from menu) $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* - $(trace-gen) changelog-d doc/manual/rl-next > $@ + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md $(trace-gen) true diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 855900d7a8b..3291d5a207a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -257,4 +257,4 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master ### Build process Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. -Non-releases build the release notes on the fly. +Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. diff --git a/flake.nix b/flake.nix index ca37948a7cb..db913f06255 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,9 @@ officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -196,7 +199,7 @@ ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease) changelog-d + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d ; buildDeps = @@ -734,8 +737,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if it's an official release - ++ lib.optional officialRelease changelog-d + # We want changelog-d in the shell even if the current build doesn't need it + ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d ; buildInputs = buildDeps ++ propagatedDeps From c5d49ec7ab7b9fb33f0336a909ac837e208be807 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:18:27 +0100 Subject: [PATCH 21/29] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 1b3c0b6d570..f120d3b5f24 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700342017, - "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "decdf666c833a325cb4417041a90681499e06a41", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { From e7e21aa0c839460a62456fa44f31339c187077ff Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:27:10 +0100 Subject: [PATCH 22/29] flake.nix: Use top level changelog-d It is about 2 MB now, as only it and libffi (tiny) are new in the build or shell closures. --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index db913f06255..a239226c07e 100644 --- a/flake.nix +++ b/flake.nix @@ -176,9 +176,7 @@ "--enable-internal-api-docs" ]; - # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - - # vastly improves output closure, and adds shell completions - changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + changelog-d = pkgs.buildPackages.changelog-d; nativeBuildDeps = [ From 2b7016cc56d12e67de9f1f25b18311866a26a5fe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 08:33:03 +0100 Subject: [PATCH 23/29] add path based redirects up to now, those were managed outside of this repo, which as unsurprisingly a real hassle to deal with if one wanted to prevent URLs from breaking when moving pages around. this change removes a large part of the friction involved in moving content in the Nix manual. possible next steps for further automation: - check for content that moved and warn if it's not reachable from links that were valid prior to a change - create redirect rules automatically based on this information --- doc/manual/_redirects | 30 ++++++++++++++++++++++++++++++ doc/manual/redirects.js | 8 +++++--- 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 doc/manual/_redirects diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 00000000000..4ea289d86ec --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,30 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - always append new redirects to the end of the file +# - redirects that should have been there but are missing can be inserted where they belong + +/expressions/expression-language /language/ 301! +/expressions/language-values /language/values 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/* /language/:splat 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! + +/package-management/channels* /command-ref/nix-channel 301! + +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d1b10109d5f..3b507adf341 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix From f56401a114cb5504c3d74b893ce270ed28fd03e3 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sat, 25 Nov 2023 11:26:57 +0800 Subject: [PATCH 24/29] `nix flake update` add deprecation warnings. This builds on #8817, to add additional UX help for people with existing muscle memory (or shell history) with --update-input and tries to gently guide them towards the newly evolved CLI UI. Co-authored-by: Cole Helbling --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++++ src/nix/flake.cc | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1c6103020f7..68287b445f3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; + addFlag({ + .longName = "recreate-lock-file", + .description = "Recreate the flake's lock file from scratch.", + .category = category, + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} + }); + addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); + addFlag({ + .longName = "update-input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, + .labels = {"input-path"}, + .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 38938f09ead..e0c67fdfa44 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -89,7 +89,13 @@ struct CmdFlakeUpdate : FlakeCommand .label="inputs", .optional=true, .handler={[&](std::string inputToUpdate){ - auto inputPath = flake::parseInputPath(inputToUpdate); + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } if (lockFlags.inputUpdates.contains(inputPath)) warn("Input '%s' was specified multiple times. You may have done this by accident."); lockFlags.inputUpdates.insert(inputPath); From 75134b7513eb781074969fc8d6d865cc95063444 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 27 Nov 2023 08:56:24 +0000 Subject: [PATCH 25/29] libexpr: add missing dependency on 'flake/call-flake.nix.gen.hh' Without the change build for `eval.o` fails occasionally as: $ make src/libexpr/eval.o GEN Makefile.config GEN src/libexpr/primops/derivation.nix.gen.hh GEN src/libexpr/fetchurl.nix.gen.hh GEN src/libexpr/parser-tab.cc GEN src/libexpr/lexer-tab.cc src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched CXX src/libexpr/eval.o src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory 519 | #include "flake/call-flake.nix.gen.hh" | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1 Noticed in https://github.com/NixOS/nixpkgs/pull/269439 --- src/libexpr/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf94904c..637f998b6d5 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From e986d20bedfc054663632255388bbd33fec99114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:25:18 +0100 Subject: [PATCH 26/29] Remove an obsolete comment --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1f7e7b2af43..887ef93b4cd 100644 --- a/flake.nix +++ b/flake.nix @@ -500,8 +500,6 @@ }); boost-nix = final.boost.override { - # enableIcu arg is not yet supported - # but will be with next nixpkgs update enableIcu = false; }; From 384ffb4443fd47d04f36c6bcc6ebf476274673ab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 09:18:56 +0100 Subject: [PATCH 27/29] add deprecation warnings in documentation this is hacky, but can serve as a stopgap until we can do it programmatically. --- src/libcmd/installables.cc | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 68287b445f3..6e670efeac0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .handler = {[&]() { lockFlags.recreateLockFile = true; @@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { From f7bfec2806708573798d610cda101f27a24d9218 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:18:29 +0100 Subject: [PATCH 28/29] maintainers/release-notes: Improve DATE check Co-authored-by: Valentin Gagarin --- maintainers/release-notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-notes b/maintainers/release-notes index 2e1be64a4f2..34cd85a5692 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -134,7 +134,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [0-9]*-[0-9]*-[0-9]*) + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" From 68c48756fece5aee77f9b44607afa9248d75e67c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:50:45 +0100 Subject: [PATCH 29/29] libexpr/local.mk: Make eval compile deps regular Dependency is now entirely through the eval.cc rule. All gen.hh deps are now there. --- src/libexpr/local.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 637f998b6d5..b37fe6f1d80 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh - -$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =