Merge remote-tracking branch 'origin/master' into lazy-trees

Also fix the handling of relative CLI overrides (e.g. --override-input
B/C "path:./flakeB/flakeD").

.github/CODEOWNERS

@@ -14,5 +14,12 @@
 /doc @fricklerhandwerk
 *.md @fricklerhandwerk
 
+# Documentation of built-in functions
+src/libexpr/primops.cc @fricklerhandwerk @roberth
+# Documentation on experimental features
+src/libutil/experimental-features.cc @fricklerhandwerk
+# Documentation on configuration settings
+src/libstore/globals.hh @fricklerhandwerk
+
 # Libstore layer
 /src/libstore @thufschmitt

.gitignore

@@ -21,12 +21,14 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
@@ -144,3 +146,6 @@ result
 
 # clangd and possibly more
 .cache/
+
+# Mac OS
+.DS_Store

doc/manual/_redirects

@@ -0,0 +1,30 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - always append new redirects to the end of the file
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/expressions/expression-language /language/ 301!
+/expressions/language-values /language/values 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/* /language/:splat 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+
+/package-management/channels* /command-ref/nix-channel 301!
+
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+

doc/manual/local.mk

@@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
@@ -144,6 +144,24 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
+
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -177,7 +195,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// redirect rules for URL fragments (client-side) to prevent link rot.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix

doc/manual/rl-next/config

@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix

doc/manual/rl-next/mounted-ssh-store.md

@@ -0,0 +1,9 @@
+synopsis: Mounted SSH Store
+issues: #7890
+prs: #7912
+description: {
+
+Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+}

doc/manual/rl-next/nix-env-json-drv-path.md

@@ -0,0 +1,9 @@
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: #9257
+description: {
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+}
+
+

doc/manual/src/SUMMARY.md.in

@@ -114,7 +114,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

doc/manual/src/contributing/hacking.md

@@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+synopsis: Basically a title
+issues: #1234
+prs: #1238
+description: {
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+
+}
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.

doc/manual/src/language/values.md

@@ -116,6 +116,7 @@
   [store path]: ../glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
+
   [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions
 
   At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.

flake.nix

@@ -14,6 +14,9 @@
 
       officialRelease = false;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -154,7 +157,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
-            "--with-boost=${boost}/lib"
+            "--with-boost=${boost-nix}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
           ]
           ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
@@ -173,6 +176,8 @@
           "--enable-internal-api-docs"
         ];
 
+        changelog-d = pkgs.buildPackages.changelog-d;
+
         nativeBuildDeps =
           [
             buildPackages.bison
@@ -190,7 +195,10 @@
             buildPackages.jq # Also for custom mdBook preprocessor.
             buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
+          # Official releases don't have rl-next, so we don't need to compile a changelog
+          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
+          ;
 
         buildDeps =
           [ curl
@@ -202,7 +210,7 @@
               version = libgit2.lastModifiedDate;
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
-            boost
+            boost-nix
             lowdown-nix
             libsodium
           ]
@@ -423,14 +431,14 @@
 
             propagatedBuildInputs = propagatedDeps;
 
-            disallowedReferences = [ boost ];
+            disallowedReferences = [ boost-nix ];
 
             preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
-                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
                 rm -f $out/lib/*.a
                 ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
@@ -440,9 +448,9 @@
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
                   done
-                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
                 ''}
               '';
 
@@ -470,9 +478,13 @@
               ''}
               ${lib.optionalString currentStdenv.isDarwin ''
               install_name_tool \
-                -change ${boost}/lib/libboost_context.dylib \
+                -change ${boost-nix}/lib/libboost_context.dylib \
                 $out/lib/libboost_context.dylib \
                 $out/lib/libnixutil.dylib
+              install_name_tool \
+                -change ${boost-nix}/lib/libboost_regex.dylib \
+                $out/lib/libboost_regex.dylib \
+                $out/lib/libnixexpr.dylib
               ''}
             '';
 
@@ -495,6 +507,10 @@
             meta.mainProgram = "nix";
           });
 
+          boost-nix = final.boost.override {
+            enableIcu = false;
+          };
+
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
             name = "lowdown-0.9.0";
 
@@ -727,6 +743,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
+              # We want changelog-d in the shell even if the current build doesn't need it
+              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps