diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml deleted file mode 100644 index 2a75747479f..00000000000 --- a/.github/workflows/hydra_status.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Hydra status - -permissions: read-all - -on: - schedule: - - cron: "12,42 * * * *" - workflow_dispatch: - -jobs: - check_hydra_status: - name: Check Hydra status - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: bash scripts/check-hydra-status.sh - diff --git a/build-utils-meson/generate-header/meson.build b/build-utils-meson/generate-header/meson.build new file mode 100644 index 00000000000..dfbe1375f4f --- /dev/null +++ b/build-utils-meson/generate-header/meson.build @@ -0,0 +1,7 @@ +bash = find_program('bash', native: true) + +gen_header = generator( + bash, + arguments : [ '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], + output : '@PLAINNAME@.gen.hh', +) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index ba5667a4305..90eaa1a73fe 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -116,9 +116,12 @@ let storeInfo = commandInfo.stores; inherit inlineHTML; }; + hasInfix = infix: content: + builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content); in optionalString (details ? doc) ( - if match ".*@store-types@.*" details.doc != null + # An alternate implementation with builtins.match stack overflowed on some systems. + if hasInfix "@store-types@" details.doc then help-stores else details.doc ); diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md new file mode 100644 index 00000000000..05023697c96 --- /dev/null +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -0,0 +1,53 @@ +--- +synopsis: "`nix-repl`'s `:doc` shows documentation comments" +significance: significant +issues: +- 3904 +- 10771 +prs: +- 1652 +- 9054 +- 11072 +--- + +`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. +This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + +Example: + +``` +nix-repl> :doc lib.toFunction +Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + +Inputs + + v + + : Any value + +Examples + + :::{.example} + +## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: +``` + +Known limitations: +- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. +- Some extensions to markdown are not yet supported, as you can see in the example above. + +We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. + +[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md new file mode 100644 index 00000000000..c887a598a05 --- /dev/null +++ b/doc/manual/rl-next/shebang-relative.md @@ -0,0 +1,62 @@ +--- +synopsis: "`nix-shell` shebang uses relative path" +prs: +- 5088 +- 11058 +issues: +- 4232 +--- + + +Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory). +Previously they were resolved relative to the current working directory. + +For example, consider the following script in `~/myproject/say-hi`: + +```shell +#!/usr/bin/env nix-shell +#!nix-shell --expr 'import ./shell.nix' +#!nix-shell --arg toolset './greeting-tools.nix' +#!nix-shell -i bash +hello +``` + +Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: + +```console +[hostname:~]$ ./myproject/say-hi +error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist +``` + +Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + +```console +$ ./myproject/say-hi +Hello, world! +``` + +**Opt-out** + +This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. +The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. +This option will be removed in a future release. + +**`nix` command shebang** + +The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. + +Example: + +```shell +#!/usr/bin/env nix +#!nix develop +#!nix --expr ``import ./shell.nix`` +#!nix -c bash +hello +``` diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index a96ba997bed..3949164d503 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -91,7 +91,7 @@ A environment variables that Google Test accepts are also worth knowing: Putting the two together, one might run ```bash -GTEST_BREIF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v ``` for short but comprensive output. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 238c502f979..b0779ea95b4 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -190,18 +190,13 @@ This section covers syntax and semantics of the Nix language. ### Path {#path-literal} - *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`. - - Paths are suitable for referring to local files, and are often preferable over strings. - - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal. - - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). - - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation. - - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + *Paths* can be expressed by path literals such as `./builder.sh`. A path literal must contain at least one slash to be recognised as such. For instance, `builder.sh` is not a path: it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). Path literals may also refer to absolute paths by starting with a slash. > **Note** @@ -215,15 +210,6 @@ This section covers syntax and semantics of the Nix language. For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. - Paths can be used in [string interpolation] and string concatenation. - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed. - - [store path]: @docroot@/store/store-path.md - Path literals can also include [string interpolation], besides being [interpolated into other expressions]. [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md index c6cfb3c6909..229756e6be2 100644 --- a/doc/manual/src/language/types.md +++ b/doc/manual/src/language/types.md @@ -50,8 +50,37 @@ The function [`builtins.isString`](builtins.md#builtins-isString) can be used to ### Path {#type-path} - +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. ### Null {#type-null} diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 24ec7ae14aa..5cff05d66ce 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -41,4 +41,30 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/flake.nix b/flake.nix index d83c2ecad36..51dbc309108 100644 --- a/flake.nix +++ b/flake.nix @@ -324,6 +324,7 @@ ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ [ pkgs.buildPackages.cmake + pkgs.shellcheck modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 8f95e788bec..b5c7bfd530b 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -143,6 +143,7 @@ ''^src/libstore/common-protocol-impl\.hh$'' ''^src/libstore/common-protocol\.cc$'' ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' ''^src/libstore/content-address\.cc$'' ''^src/libstore/content-address\.hh$'' ''^src/libstore/daemon\.cc$'' @@ -215,7 +216,6 @@ ''^src/libstore/serve-protocol\.hh$'' ''^src/libstore/sqlite\.cc$'' ''^src/libstore/sqlite\.hh$'' - ''^src/libstore/ssh-store-config\.hh$'' ''^src/libstore/ssh-store\.cc$'' ''^src/libstore/ssh\.cc$'' ''^src/libstore/ssh\.hh$'' @@ -495,11 +495,9 @@ excludes = [ # We haven't linted these files yet ''^config/install-sh$'' - ''^misc/systemv/nix-daemon$'' ''^misc/bash/completion\.sh$'' ''^misc/fish/completion\.fish$'' ''^misc/zsh/completion\.zsh$'' - ''^scripts/check-hydra-status\.sh$'' ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' diff --git a/meson.build b/meson.build index 356d978dc8e..1c46c5c2881 100644 --- a/meson.build +++ b/meson.build @@ -6,6 +6,7 @@ project('nix-dev-shell', 'cpp', subproject_dir : 'src', ) +# Internal Libraries subproject('libutil') subproject('libstore') subproject('libfetchers') @@ -14,14 +15,18 @@ subproject('libflake') subproject('libmain') subproject('libcmd') +# Executables +subproject('nix') + # Docs subproject('internal-api-docs') subproject('external-api-docs') -# C wrappers +# External C wrapper libraries subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libmain-c') # Language Bindings subproject('perl') diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea53716721..e8326f9472b 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/packaging/components.nix b/packaging/components.nix index f1cd3b9c6d3..870e9ae615f 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -29,9 +29,13 @@ in nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; nix-cmd = callPackage ../src/libcmd/package.nix { }; + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 34b3449718d..73ba9cd586f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -11,11 +11,28 @@ versionSuffix, }: +let + prevStdenv = stdenv; +in + let inherit (pkgs) lib; root = ../.; + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. resolvePath = p: p; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 0bbbc31f77b..dbe99247675 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -52,7 +52,9 @@ let "nix-flake" "nix-flake-tests" "nix-main" + "nix-main-c" "nix-cmd" + "nix-ng" ]; in { diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e9498..00000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 582e6d623d0..a0a404e575a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d7e..905e70f32c9 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE - -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0eefb..dc78d3e599d 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 74d146c66c5..67fef190920 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -132,7 +133,7 @@ ref EvalCommand::getEvalState() #else std::make_shared( #endif - lookupPath, getEvalStore(), evalSettings, getStore()) + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 37b3905a861..25b611a4884 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" @@ -7,6 +8,7 @@ #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" @@ -16,6 +18,10 @@ namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + EvalSettings evalSettings { settings.readOnlyMode, { @@ -24,7 +30,7 @@ EvalSettings evalSettings { [](ref store, std::string_view rest) { experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. - auto flakeRef = parseFlakeRef(std::string { rest }, {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto [accessor, _] = flakeRef.resolve(store).lazyFetch(store); return SourcePath(accessor); @@ -35,6 +41,12 @@ EvalSettings evalSettings { static GlobalConfig::Register rEvalSettings(&evalSettings); + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + CompatibilitySettings compatibilitySettings {}; static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); @@ -171,8 +183,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -202,7 +214,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) auto v = state.allocValue(); std::visit(overloaded { [&](const AutoArgExpr & arg) { - state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath("."))); + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); }, [&](const AutoArgString & arg) { v->mkString(arg.s); @@ -230,7 +242,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, _] = flakeRef.resolve(state.store).lazyFetch(state.store); return SourcePath(accessor); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 8d303ee7c13..c62365b32e2 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -11,17 +11,32 @@ namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; struct EvalSettings; struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + /** * @todo Get rid of global setttings variables */ extern EvalSettings evalSettings; +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + /** * Settings that control behaviors that have changed since Nix 2.3. */ diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh index 5dc0eaf2b38..a129a957a64 100644 --- a/src/libcmd/compatibility-settings.hh +++ b/src/libcmd/compatibility-settings.hh @@ -7,12 +7,29 @@ struct CompatibilitySettings : public Config CompatibilitySettings() = default; + // Added in Nix 2.24, July 2024. Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. - You may set this to `false` to revert to the Nix 2.3 behavior. + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. )"}; }; diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d42fa7aaccc..8796ad5ba79 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -210,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c140d..8e0a232ef8a 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392bb5..798cb5e1a00 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6835c512c1c..0fe956ec023 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,6 +27,8 @@ #include +#include "strings-inline.hh" + namespace nix { void completeFlakeInputPath( @@ -129,7 +131,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -170,14 +172,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -289,10 +292,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s if (v2.type() == nAttrs) { for (auto & i : *v2.attrs()) { - std::string name = state->symbols[i.name]; + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +341,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +376,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +408,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -534,7 +539,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -851,6 +857,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -873,6 +880,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 2c8a9fa3374..687b37aacbd 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -64,6 +64,7 @@ add_project_arguments( '-include', 'config-util.hh', '-include', 'config-store.hh', # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', '-include', 'config-main.hh', '-include', 'config-cmd.hh', language : 'cpp', diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 420cce1db89..b285c8a9ae5 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -73,7 +73,7 @@ static int listPossibleCallback(char * s, char *** avp) { auto possible = curRepl->completePrefix(s); - if (possible.size() > (INT_MAX / sizeof(char *))) + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) throw Error("too many completions"); int ac = 0; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index ce1c5af6997..b5d0816dd2c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include "repl-interacter.hh" #include "repl.hh" @@ -10,8 +9,6 @@ #include "shared.hh" #include "config-global.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" @@ -29,12 +26,16 @@ #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { /** @@ -616,6 +617,38 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + + } else { + evalString(arg, v); + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -633,6 +666,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -690,14 +736,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk index 51b02562e12..227a4095b30 100644 --- a/src/libexpr-c/local.mk +++ b/src/libexpr-c/local.mk @@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ $(INCLUDE_libstore) $(INCLUDE_libstorec) \ $(INCLUDE_libexpr) $(INCLUDE_libexprc) -libexprc_LIBS = libutil libutilc libstore libstorec libexpr +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr libexprc_LDFLAGS += $(THREAD_LDFLAGS) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 13e0f5f3a9f..547453f8f93 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -1,12 +1,10 @@ #include -#include #include #include -#include "config.hh" #include "eval.hh" +#include "eval-gc.hh" #include "globals.hh" -#include "util.hh" #include "eval-settings.hh" #include "nix_api_expr.h" @@ -112,12 +110,14 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c static_cast(alignof(EvalState))); auto * p2 = static_cast(p); new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, .settings = nix::EvalSettings{ nix::settings.readOnlyMode, }, .state = nix::EvalState( nix::LookupPath::parse(lookupPath), store->ptr, + p2->fetchSettings, p2->settings), }; loadConfFile(p2->settings); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index d4ccffd29d6..12f24b6eb07 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,7 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include "fetch-settings.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-set.hh" @@ -8,6 +9,7 @@ struct EvalState { + nix::fetchers::Settings fetchSettings; nix::EvalSettings settings; nix::EvalState state; }; diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3c3dd6ca99d..3565092a4a6 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -115,7 +115,7 @@ class NixCExternalValue : public nix::ExternalValueBase /** * Compare to another value of the same type. */ - virtual bool operator==(const ExternalValueBase & b) const override + virtual bool operator==(const ExternalValueBase & b) const noexcept override { if (!desc.equal) { return false; diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 2f2f99617e2..cb5d9ee8928 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -383,7 +383,7 @@ nix_value * nix_get_attr_byidx( try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); + *name = state->state.symbols[a.name].c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); return as_nix_value_ptr(a.value); @@ -399,7 +399,7 @@ nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalSt try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - return ((const std::string &) (state->state.symbols[a.name])).c_str(); + return state->state.symbols[a.name].c_str(); } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9ad201b63ba..d61d9363070 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -76,7 +76,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!a) { std::set attrNames; for (auto & attr : *v->attrs()) - attrNames.insert(state.symbols[attr.name]); + attrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index ba798196d3c..4df9a1acdc9 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -5,7 +5,6 @@ #include "symbol-table.hh" #include -#include namespace nix { @@ -28,9 +27,9 @@ struct Attr Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name), pos(pos), value(value) { }; Attr() { }; - bool operator < (const Attr & a) const + auto operator <=> (const Attr & a) const { - return name < a.name; + return name <=> a.name; } }; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 5a3b32fadac..da0d789b436 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -95,7 +95,7 @@ struct AttrDb { try { auto state(_state->lock()); - if (!failed) + if (!failed && state->txn->active) state->txn->commit(); state->txn.reset(); } catch (...) { @@ -225,7 +225,7 @@ struct AttrDb (key.first) (symbols[key.second]) (AttrType::ListOfStrings) - (concatStringsSep("\t", l)).exec(); + (dropEmptyInitThenConcatStringsSep("\t", l)).exec(); return state->db.getLastInsertedRowId(); }); @@ -435,12 +435,12 @@ std::vector AttrCursor::getAttrPath(Symbol name) const std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() @@ -484,7 +484,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(root->state.symbols[name]); + strAttrNames.insert(std::string(root->state.symbols[name])); return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index fe48e054bf2..2f604647331 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -1,7 +1,5 @@ #pragma once -#include - #include "error.hh" #include "pos-idx.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 6b7b52cef32..eb5761638a7 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,5 +1,4 @@ #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" @@ -57,7 +56,7 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo builtinsAbortOnWarn = true; } -Strings EvalSettings::getDefaultNixPath() const +Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { @@ -70,11 +69,9 @@ Strings EvalSettings::getDefaultNixPath() const } }; - if (!restrictEval && !pureEval) { - add(getNixDefExpr() + "/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); return res; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index af9725d96c0..55545c8920f 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -3,6 +3,7 @@ #include "config.hh" #include "source-path.hh" +#include "ref.hh" namespace nix { @@ -40,7 +41,7 @@ struct EvalSettings : Config bool & readOnlyMode; - Strings getDefaultNixPath() const; + static Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6cbc2e32d1a..a023153cd00 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,6 @@ #include "eval.hh" +#include "eval-gc.hh" #include "eval-settings.hh" -#include "hash.hh" #include "primops.hh" #include "print-options.hh" #include "exit.hh" @@ -16,7 +16,6 @@ #include "print.hh" #include "filtering-source-accessor.hh" #include "memory-source-accessor.hh" -#include "signals.hh" #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" @@ -24,7 +23,6 @@ #include "parser-tab.hh" #include -#include #include #include #include @@ -51,6 +49,8 @@ #endif +#include "strings-inline.hh" + using json = nlohmann::json; namespace nix { @@ -222,9 +222,11 @@ static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore) - : settings{settings} + : fetchSettings{fetchSettings} + , settings{settings} , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -563,6 +565,54 @@ std::optional EvalState::getDoc(Value & v) .doc = doc, }; } + if (v.isLambda()) { + auto exprLambda = v.payload.lambda.fun; + + std::stringstream s(std::ios_base::out); + std::string name; + auto pos = positions[exprLambda->getPos()]; + std::string docStr; + + if (exprLambda->name) { + name = symbols[exprLambda->name]; + } + + if (exprLambda->docComment) { + docStr = exprLambda->docComment.getInnerText(positions); + } + + if (name.empty()) { + s << "Function "; + } + else { + s << "Function `" << name << "`"; + if (pos) + s << "\\\n … " ; + else + s << "\\\n"; + } + if (pos) { + s << "defined at " << pos; + } + if (!docStr.empty()) { + s << "\n\n"; + } + + s << docStr; + + s << '\0'; // for making a c string below + std::string ss = s.str(); + + return Doc { + .pos = pos, + .name = name, + .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... + .args = {}, + .doc = + // FIXME: this leaks; make the field std::string? + strdup(ss.data()), + }; + } return {}; } @@ -639,11 +689,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) - vm[st[j.name]] = j.value; + vm.insert_or_assign(std::string(st[j.name]), j.value); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm[st[i.first]] = env.values[i.second]; + vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); } } } @@ -1027,7 +1077,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) if (!e) e = parseExprFromFile(resolvedPath); - fileParseCache[resolvedPath] = e; + fileParseCache.emplace(resolvedPath, e); try { auto dts = debugRepl @@ -1050,8 +1100,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) throw; } - fileEvalCache[resolvedPath] = v; - if (path != resolvedPath) fileEvalCache[path] = v; + fileEvalCache.emplace(resolvedPath, v); + if (path != resolvedPath) fileEvalCache.emplace(path, v); } @@ -1354,7 +1404,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; for (auto & attr : *vAttrs->attrs()) - allAttrNames.insert(state.symbols[attr.name]); + allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); state.error("attribute '%1%' missing", state.symbols[name]) .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); @@ -1381,6 +1431,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) v = *vAttrs; } +Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) +{ + Value vTmp; + Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + + if (attrPath.size() == 1) { + e->eval(state, env, vTmp); + } else { + ExprSelect init(*this); + init.attrPath.pop_back(); + init.eval(state, env, vTmp); + } + attrs = vTmp; + return name; +} + void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) { @@ -1512,7 +1578,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(symbols[formal.name]); + formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), @@ -2903,13 +2969,37 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); + DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath + DocCommentMap *docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.try_emplace(*sourcePath); + docComments = &it->second; + } + + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); result->bindVars(*this, staticEnv); return result; } +DocComment EvalState::getDocCommentForPos(PosIdx pos) +{ + auto pos2 = positions[pos]; + auto path = pos2.getSourcePath(); + if (!path) + return {}; + + auto table = positionToDocComment.find(*path); + if (table == positionToDocComment.end()) + return {}; + + auto it = table->second.find(pos); + if (it == table->second.end()) + return {}; + return it->second; +} std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { @@ -2919,7 +3009,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & } -bool ExternalValueBase::operator==(const ExternalValueBase & b) const +bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept { return false; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 55879aa867d..d8daaac6b12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,21 +3,21 @@ #include "attr-set.hh" #include "eval-error.hh" -#include "eval-gc.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "position.hh" +#include "pos-table.hh" #include "source-accessor.hh" #include "search-path.hh" #include "repl-exit-status.hh" +#include "ref.hh" #include #include -#include -#include #include namespace nix { @@ -30,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +namespace fetchers { struct Settings; } struct EvalSettings; class EvalState; class StorePath; @@ -45,7 +46,7 @@ namespace eval_cache { /** * Function that implements a primop. */ -typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v); /** * Info about a primitive operation, and its implementation @@ -86,7 +87,7 @@ struct PrimOp /** * Implementation of the primop. */ - std::function::type> fun; + std::function fun; /** * Optional experimental for this to be gated on. @@ -131,6 +132,8 @@ struct Constant typedef std::map ValMap; #endif +typedef std::unordered_map DocCommentMap; + struct Env { Env * up; @@ -164,6 +167,7 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const fetchers::Settings & fetchSettings; const EvalSettings & settings; SymbolTable symbols; PosTable positions; @@ -311,15 +315,15 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - Sync> srcToStore; + Sync> srcToStore; /** * A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::unordered_map FileParseCache; #endif FileParseCache fileParseCache; @@ -327,12 +331,18 @@ private: * A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::unordered_map FileEvalCache; #endif FileEvalCache fileEvalCache; + /** + * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. + * Grouped by file. + */ + std::unordered_map positionToDocComment; + LookupPath lookupPath; std::map> lookupPathResolved; @@ -359,6 +369,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); @@ -818,6 +829,8 @@ public: const SourcePath & path, PosIdx pos); + DocComment getDocCommentForPos(PosIdx pos); + private: /** diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 8967334237a..7041a3932ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -342,9 +342,9 @@ std::optional getDerivation(EvalState & state, Value & v, } -static std::string addToPath(const std::string & s1, const std::string & s2) +static std::string addToPath(const std::string & s1, std::string_view s2) { - return s1.empty() ? s2 : s1 + "." + s2; + return s1.empty() ? std::string(s2) : s1 + "." + s2; } diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc new file mode 100644 index 00000000000..d9eeb73e269 --- /dev/null +++ b/src/libexpr/lexer-helpers.cc @@ -0,0 +1,30 @@ +#include "lexer-tab.hh" +#include "lexer-helpers.hh" +#include "parser-tab.hh" + +void nix::lexer::internal::initLoc(YYLTYPE * loc) +{ + loc->beginOffset = loc->endOffset = 0; +} + +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +{ + loc->stash(); + + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset; + ParserLocation docEnd; + docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + + loc->beginOffset = loc->endOffset; + loc->endOffset += len; +} diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh new file mode 100644 index 00000000000..caba6e18f48 --- /dev/null +++ b/src/libexpr/lexer-helpers.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix::lexer::internal { + +void initLoc(YYLTYPE * loc); + +void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); + +} // namespace nix::lexer diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 8c0f9d1f2f7..58401be8e71 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -5,7 +5,7 @@ %option stack %option nodefault %option nounput noyy_top_state - +%option extra-type="::nix::LexerState *" %s DEFAULT %x STRING @@ -14,6 +14,10 @@ %x INPATH_SLASH %x PATH_START +%top { +#include "parser-tab.hh" // YYSTYPE +#include "parser-state.hh" +} %{ #ifdef __clang__ @@ -22,27 +26,18 @@ #include "nixexpr.hh" #include "parser-tab.hh" - -using namespace nix; +#include "lexer-helpers.hh" namespace nix { - -#define CUR_POS state->at(*yylloc) - -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 0; - loc->first_column = loc->last_column = 0; + struct LexerState; } -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - loc->stash(); +using namespace nix; +using namespace nix::lexer::internal; - loc->first_column = loc->last_column; - loc->last_column += len; -} +namespace nix { +#define CUR_POS state->at(*yylloc) // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. @@ -79,7 +74,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -279,9 +274,32 @@ or { return OR_KW; } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } {URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ +%{ +// Doc comment rule +// +// \/\*\* /** +// [^/*] reject /**/ (empty comment) and /*** +// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule +// ( )* zero or more non-ending sequences +// \* end(1) +// \/ end(2) +%} +\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { + LexerState & lexerState = *yyget_extra(yyscanner); + lexerState.docCommentDistance = 0; + lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset; + lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset; +} + + +%{ +// The following rules have docCommentDistance-- +// This compensates for the docCommentDistance++ which happens by default to +// make all the other rules invalidate the doc comment. +%} +[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; } +\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; } +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; } {ANY} { /* Don't return a negative number, as this will cause diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 9fe7c17c4e5..fa90e7b41a3 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -69,6 +69,7 @@ add_project_arguments( '-include', 'config-util.hh', '-include', 'config-store.hh', # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', language : 'cpp', ) @@ -116,20 +117,17 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) +subdir('build-utils-meson/generate-header') + generated_headers = [] foreach header : [ 'imported-drv-to-derivation.nix', 'fetchurl.nix', 'call-flake.nix', ] - generated_headers += custom_target( - command : [ 'bash', '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], - input : header, - output : '@PLAINNAME@.gen.hh', - ) + generated_headers += gen_header.process(header) endforeach - sources = files( 'attr-path.cc', 'attr-set.cc', @@ -141,6 +139,7 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', + 'lexer-helpers.cc', 'nixexpr.cc', 'paths.cc', 'primops.cc', @@ -167,6 +166,7 @@ headers = [config_h] + files( 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', + # internal: 'lexer-helpers.hh', 'nixexpr.hh', 'parser-state.hh', 'pos-idx.hh', diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 8ab701638b7..cb1501f3ffa 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -1,5 +1,4 @@ #include "nixexpr.hh" -#include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" #include "util.hh" @@ -8,6 +7,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { unsigned long Expr::nrExprs = 0; @@ -81,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co return sa < sb; }); std::vector inherits; - std::map> inheritsFrom; + // We can use the displacement as a proxy for the order in which the symbols were parsed. + // The assignment of displacements should be deterministic, so that showBindings is deterministic. + std::map> inheritsFrom; for (auto & i : sorted) { switch (i->second.kind) { case AttrDef::Kind::Plain: @@ -92,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co case AttrDef::Kind::InheritedFrom: { auto & select = dynamic_cast(*i->second.e); auto & from = dynamic_cast(*select.e); - inheritsFrom[&from].push_back(i->first); + inheritsFrom[from.displ].push_back(i->first); break; } } @@ -104,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co } for (const auto & [from, syms] : inheritsFrom) { str << "inherit ("; - (*inheritFromExprs)[from->displ]->show(symbols, str); + (*inheritFromExprs)[from]->show(symbols, str); str << ")"; for (auto sym : syms) str << " " << symbols[sym]; str << "; "; @@ -582,6 +585,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const return fmt("%1% at %2%", id, state.positions[pos]); } +void ExprLambda::setDocComment(DocComment docComment) { + // RFC 145 specifies that the innermost doc comment wins. + // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + if (!this->docComment) { + this->docComment = docComment; + + // Curried functions are defined by putting a function directly + // in the body of another function. To render docs for those, we + // need to propagate the doc comment to the innermost function. + // + // If we have our own comment, we've already propagated it, so this + // belongs in the same conditional. + body->setDocComment(docComment); + } +}; + /* Position table. */ @@ -626,4 +645,22 @@ size_t SymbolTable::totalSize() const return n; } +std::string DocComment::getInnerText(const PosTable & positions) const { + auto beginPos = positions[begin]; + auto endPos = positions[end]; + auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or(""); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + return docStr; +} + } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index bc491395e4a..648fc83eeda 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -6,21 +6,58 @@ #include "value.hh" #include "symbol-table.hh" -#include "error.hh" -#include "position.hh" #include "eval-error.hh" #include "pos-idx.hh" -#include "pos-table.hh" namespace nix { - -struct Env; -struct Value; class EvalState; +class PosTable; +struct Env; struct ExprWith; struct StaticEnv; +struct Value; + +/** + * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md) + * + * Note that this does not implement the following: + * - argument attribute names ("formals"): TBD + * - argument names: these are internal to the function and their names may not be optimal for documentation + * - function arity (degree of currying or number of ':'s): + * - Functions returning partially applied functions have a higher arity + * than can be determined locally and without evaluation. + * We do not want to present false data. + * - Some functions should be thought of as transformations of other + * functions. For instance `overlay -> overlay -> overlay` is the simplest + * way to understand `composeExtensions`, but its implementation looks like + * `f: g: final: prev: <...>`. The parameters `final` and `prev` are part + * of the overlay concept, while distracting from the function's purpose. + */ +struct DocComment { + /** + * Start of the comment, including the opening, ie `/` and `**`. + */ + PosIdx begin; + + /** + * Position right after the final asterisk and `/` that terminate the comment. + */ + PosIdx end; + + /** + * Whether the comment is set. + * + * A `DocComment` is small enough that it makes sense to pass by value, and + * therefore baking optionality into it is also useful, to avoiding the memory + * overhead of `std::optional`. + */ + operator bool() const { return static_cast(begin); } + + std::string getInnerText(const PosTable & positions) const; + +}; /** * An attribute path is a sequence of attribute names. @@ -57,6 +94,7 @@ struct Expr virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); + virtual void setDocComment(DocComment docComment) { }; virtual PosIdx getPos() const { return noPos; } }; @@ -159,6 +197,17 @@ struct ExprSelect : Expr ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } + + /** + * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. + * + * @param[out] v The attribute set that should contain the last attribute name (if it exists). + * @return The last attribute name in `attrPath` + * + * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. + */ + Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); + COMMON_METHODS }; @@ -281,6 +330,8 @@ struct ExprLambda : Expr Symbol arg; Formals * formals; Expr * body; + DocComment docComment; + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { @@ -293,6 +344,7 @@ struct ExprLambda : Expr std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } + virtual void setDocComment(DocComment docComment) override; COMMON_METHODS }; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index cff6282faa0..4bb5c92046b 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "eval.hh" namespace nix { @@ -20,25 +22,59 @@ struct StringToken struct ParserLocation { - int first_line, first_column; - int last_line, last_column; + int beginOffset; + int endOffset; // backup to recover from yyless(0) - int stashed_first_column, stashed_last_column; + int stashedBeginOffset, stashedEndOffset; void stash() { - stashed_first_column = first_column; - stashed_last_column = last_column; + stashedBeginOffset = beginOffset; + stashedEndOffset = endOffset; } void unstash() { - first_column = stashed_first_column; - last_column = stashed_last_column; + beginOffset = stashedBeginOffset; + endOffset = stashedEndOffset; } + + /** Latest doc comment position, or 0. */ + int doc_comment_first_column, doc_comment_last_column; +}; + +struct LexerState +{ + /** + * Tracks the distance to the last doc comment, in terms of lexer tokens. + * + * The lexer sets this to 0 when reading a doc comment, and increments it + * for every matched rule; see `lexer-helpers.cc`. + * Whitespace and comment rules decrement the distance, so that they result + * in a net 0 change in distance. + */ + int docCommentDistance = std::numeric_limits::max(); + + /** + * The location of the last doc comment. + * + * (stashing fields are not used) + */ + ParserLocation lastDocCommentLoc; + + /** + * @brief Maps some positions to a DocComment, where the comment is relevant to the location. + */ + std::unordered_map & positionToDocComment; + + PosTable & positions; + PosTable::Origin origin; + + PosIdx at(const ParserLocation & loc); }; struct ParserState { + const LexerState & lexerState; SymbolTable & symbols; PosTable & positions; Expr * result; @@ -255,12 +291,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(std::move(s2))); + // Ignore empty strings for a minor optimisation and AST simplification + if (s2 != "") { + es2->emplace_back(i->first, new ExprString(std::move(s2))); + } }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); } + // If there is nothing at all, return the empty string directly. + // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. + if (es2->size() == 0) { + auto *const result = new ExprString(""); + delete es2; + return result; + } + /* If this is a single string, then don't do a concatenation. */ if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { auto *const result = (*es2)[0].second; @@ -270,9 +317,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, return new ExprConcatStrings(pos, true, es2); } +inline PosIdx LexerState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.beginOffset); +} + inline PosIdx ParserState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_column); + return positions.add(origin, loc.beginOffset); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6e9146b7441..a5de2d42ee6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,8 +31,25 @@ #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +// For efficiency, we only track offsets; not line,column coordinates +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \ + (Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \ + } \ + else \ + { \ + (Current).beginOffset = (Current).endOffset = \ + YYRHSLOC (Rhs, 0).endOffset; \ + } \ + while (0) + namespace nix { +typedef std::unordered_map DocCommentMap; + Expr * parseExprFromBuf( char * text, size_t length, @@ -41,6 +58,7 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -65,8 +83,7 @@ using namespace nix; void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { - loc->first_column = loc->last_column; - loc->first_line = loc->last_line; + loc->beginOffset = loc->endOffset; } throw ParseError({ .msg = HintFmt(error), @@ -74,6 +91,14 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * }); } +#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos)) +static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) { + auto it = lexerState.positionToDocComment.find(start); + if (it != lexerState.positionToDocComment.end()) { + lambda->setDocComment(it->second); + } +} + %} @@ -124,6 +149,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS + %right IMPL %left OR %left AND @@ -145,18 +171,28 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } + { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } + { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + $$ = me; + SET_DOC_POS(me, @1); + } | '{' formals '}' '@' ID ':' expr_function { auto arg = state->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ID '@' '{' formals '}' ':' expr_function { auto arg = state->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + $$ = me; + SET_DOC_POS(me, @1); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -325,7 +361,22 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } + : binds attrpath '=' expr ';' { + $$ = $1; + + auto pos = state->at(@2); + auto exprPos = state->at(@4); + { + auto it = state->lexerState.positionToDocComment.find(pos); + if (it != state->lexerState.positionToDocComment.end()) { + $4->setDocComment(it->second); + state->lexerState.positionToDocComment.emplace(exprPos, it->second); + } + } + + state->addAttr($$, std::move(*$2), $4, pos); + delete $2; + } | binds INHERIT attrs ';' { $$ = $1; for (auto & [i, iPos] : *$3) { @@ -444,21 +495,28 @@ Expr * parseExprFromBuf( SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols) { yyscan_t scanner; + LexerState lexerState { + .positionToDocComment = docComments, + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; ParserState state { + .lexerState = lexerState, .symbols = symbols, .positions = positions, .basePath = basePath, - .origin = positions.addOrigin(origin, length), + .origin = lexerState.origin, .rootFS = rootFS, .s = astSymbols, .settings = settings, }; - yylex_init(&scanner); + yylex_init_extra(&lexerState, &scanner); Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index e94fd85c673..2faa6b7fe4f 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -1,6 +1,7 @@ #pragma once #include +#include namespace nix { @@ -8,6 +9,7 @@ class PosIdx { friend struct LazyPosAcessors; friend class PosTable; + friend class std::hash; private: uint32_t id; @@ -28,9 +30,9 @@ public: return id > 0; } - bool operator<(const PosIdx other) const + auto operator<=>(const PosIdx other) const { - return id < other.id; + return id <=> other.id; } bool operator==(const PosIdx other) const @@ -38,12 +40,25 @@ public: return id == other.id; } - bool operator!=(const PosIdx other) const + size_t hash() const noexcept { - return id != other.id; + return std::hash{}(id); } }; inline PosIdx noPos = {}; } + +namespace std { + +template<> +struct hash +{ + std::size_t operator()(nix::PosIdx pos) const noexcept + { + return pos.hash(); + } +}; + +} // namespace std diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh index 8a0a3ba86fa..ba2b91cf35e 100644 --- a/src/libexpr/pos-table.hh +++ b/src/libexpr/pos-table.hh @@ -1,10 +1,8 @@ #pragma once -#include -#include +#include #include -#include "chunked-vector.hh" #include "pos-idx.hh" #include "position.hh" #include "sync.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bd79d08acc8..82fcd68ae9d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,4 +1,3 @@ -#include "archive.hh" #include "derivations.hh" #include "downstream-placeholder.hh" #include "eval-inline.hh" @@ -730,7 +729,7 @@ static RegisterPrimOp primop_genericClosure(PrimOp { .doc = R"( `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function. - It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets: + It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets: - `startSet`: The initial list of attribute sets. @@ -1236,7 +1235,7 @@ static void derivationStrictInternal( for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = state.symbols[i->name]; + auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { @@ -1320,7 +1319,7 @@ static void derivationStrictInternal( if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, pos, context_below); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 7b5f4193a23..64e3abf2db4 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto [storePath, input2] = input.fetchToStore(state.store); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e86b934d13d..8b51b35488c 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -101,7 +101,7 @@ static void fetchTree( Value & v, const FetchTreeParams & params = FetchTreeParams{} ) { - fetchers::Input input; + fetchers::Input input { state.fetchSettings }; NixStringContext context; std::optional type; if (params.isFetchGit) type = "git"; @@ -166,7 +166,7 @@ static void fetchTree( "attribute 'name' isn’t supported in call to 'fetchTree'" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to the fetcher", @@ -179,13 +179,13 @@ static void fetchTree( if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { attrs.emplace("exportIgnore", Explicit{true}); } - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) state.error( "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(state.fetchSettings, url); } } diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index 96a1dd07ecf..f910fe23768 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -1,12 +1,7 @@ -foreach header : [ +generated_headers += gen_header.process( 'derivation.nix', -] - generated_headers += custom_target( - command : [ 'bash', '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], - input : header, - output : '@PLAINNAME@.gen.hh', - ) -endforeach + preserve_path_from: meson.project_source_root(), +) sources += files( 'context.cc', diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 2f377e5886e..bc17d6bfe26 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -299,6 +299,9 @@ class Printer output << ANSI_NORMAL; } + /** + * @note This may force items. + */ bool shouldPrettyPrintAttrs(AttrVec & v) { if (!options.shouldPrettyPrint() || v.empty()) { @@ -315,6 +318,9 @@ class Printer return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item attrsets only if they contain nested // structures. auto itemType = item->type(); @@ -371,6 +377,9 @@ class Printer } } + /** + * @note This may force items. + */ bool shouldPrettyPrintList(std::span list) { if (!options.shouldPrettyPrint() || list.empty()) { @@ -387,6 +396,9 @@ class Printer return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item lists only if they contain nested // structures. auto itemType = item->type(); diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 967a186dd59..c7a3563b040 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -30,9 +30,9 @@ public: return *s == s2; } - operator const std::string & () const + const char * c_str() const { - return *s; + return s->c_str(); } operator const std::string_view () const @@ -41,6 +41,11 @@ public: } friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); + + bool empty() const + { + return s->empty(); + } }; /** @@ -62,9 +67,8 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const Symbol other) const { return id < other.id; } + auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } - bool operator!=(const Symbol other) const { return id != other.id; } }; /** diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 936ecf07826..f8cc056161e 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict, out = json::object(); for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { try { - out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore); + out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); } catch (Error & e) { e.addTrace(state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1de8cdf848d..9734ebec498 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) +static XMLAttrs singletonAttrs(const std::string & name, std::string_view value) { XMLAttrs attrs; attrs[name] = value; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 208cab21d60..1f4d72d3926 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include "symbol-table.hh" @@ -112,7 +111,7 @@ class ExternalValueBase * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator ==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const noexcept; /** * Print the value as JSON. Defaults to unconvertable, i.e. throws an error diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 21c42567cb5..c7ed4c7af08 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,14 +1,9 @@ #include "fetch-settings.hh" -#include "config-global.hh" -namespace nix { +namespace nix::fetchers { -FetchSettings::FetchSettings() +Settings::Settings() { } -FetchSettings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - } diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 629967697da..f7cb34a0220 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -9,11 +9,11 @@ #include -namespace nix { +namespace nix::fetchers { -struct FetchSettings : public Config +struct Settings : public Config { - FetchSettings(); + Settings(); Setting accessTokens{this, {}, "access-tokens", R"( @@ -84,9 +84,14 @@ struct FetchSettings : public Config `narHash` attribute is specified, e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. )"}; -}; -// FIXME: don't use a global variable. -extern FetchSettings fetchSettings; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; +}; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2b93c5fca37..ab37e7c16ab 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,9 +35,11 @@ nlohmann::json dumpRegisterInputSchemeInfo() { return res; } -Input Input::fromURL(const std::string & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const std::string & url, bool requireTree) { - return fromURL(parseURL(url), requireTree); + return fromURL(settings, parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -49,10 +51,12 @@ static void fixupInput(Input & input) input.getLastModified(); } -Input Input::fromURL(const ParsedURL & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) { for (auto & [_, inputScheme] : *inputSchemes) { - auto res = inputScheme->inputFromURL(url, requireTree); + auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; @@ -64,7 +68,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) throw Error("input '%s' is unsupported", url.url); } -Input Input::fromAttrs(Attrs && attrs) +Input Input::fromAttrs(const Settings & settings, Attrs && attrs) { auto schemeName = ({ auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); @@ -78,7 +82,7 @@ Input Input::fromAttrs(Attrs && attrs) // but not all of them. Doing this is to support those other // operations which are supposed to be robust on // unknown/uninterpretable inputs. - Input input; + Input input { settings }; input.attrs = attrs; fixupInput(input); return input; @@ -99,7 +103,7 @@ Input Input::fromAttrs(Attrs && attrs) if (name != "type" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(settings, attrs); if (!res) return raw(); res->scheme = inputScheme; fixupInput(*res); @@ -152,7 +156,7 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::operator ==(const Input & other) const +bool Input::operator ==(const Input & other) const noexcept { return attrs == other.attrs; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 0ce30abddb0..7f0bd50f184 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -11,12 +11,16 @@ #include #include +#include "ref.hh" + namespace nix { class Store; class StorePath; struct SourceAccessor; } namespace nix::fetchers { struct InputScheme; +struct Settings; + /** * The `Input` object is generated by a specific fetcher, based on * user-supplied information, and contains @@ -28,6 +32,12 @@ struct Input { friend struct InputScheme; + const Settings * settings; + + Input(const Settings & settings) + : settings{&settings} + { } + std::shared_ptr scheme; // note: can be null Attrs attrs; @@ -37,16 +47,22 @@ public: * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromURL(const std::string & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree = true); /** * Create an `Input` from a an `Attrs`. * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromAttrs(Attrs && attrs); + static Input fromAttrs( + const Settings & settings, + Attrs && attrs); ParsedURL toURL() const; @@ -74,7 +90,7 @@ public: */ std::optional isRelative() const; - bool operator ==(const Input & other) const; + bool operator ==(const Input & other) const noexcept; bool contains(const Input & other) const; @@ -143,9 +159,13 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; + virtual std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + virtual std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const = 0; /** * What is the name of the scheme? diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 500064ceedc..ecc71ae471d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -115,10 +115,10 @@ git_oid hashToOID(const Hash & hash) return oid; } -Object lookupObject(git_repository * repo, const git_oid & oid) +Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY) { Object obj; - if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + if (git_object_lookup(Setter(obj), repo, &oid, type)) { auto err = git_error_last(); throw Error("getting Git object '%s': %s", oid, err->message); } @@ -909,6 +909,61 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); } + void createHardlink(const CanonPath & path, const CanonPath & target) override + { + std::vector pathComponents; + for (auto & c : path) + pathComponents.emplace_back(c); + + if (!prepareDirs(pathComponents, false)) return; + + // We can't just look up the path from the start of the root, since + // some parent directories may not have finished yet, so we compute + // a relative path that helps us find the right git_tree_builder or object. + auto relTarget = CanonPath(path).parent()->makeRelative(target); + + auto dir = pendingDirs.rbegin(); + + // For each ../ component at the start, go up one directory. + // CanonPath::makeRelative() always puts all .. elements at the start, + // so they're all handled by this loop: + std::string_view relTargetLeft(relTarget); + while (hasPrefix(relTargetLeft, "../")) { + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + ++dir; + relTargetLeft = relTargetLeft.substr(3); + } + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + + // Look up the remainder of the target, starting at the + // top-most `git_treebuilder`. + std::variant curDir{dir->builder.get()}; + Object tree; // needed to keep `entry` alive + const git_tree_entry * entry = nullptr; + + for (auto & c : CanonPath(relTargetLeft)) { + if (auto builder = std::get_if(&curDir)) { + assert(*builder); + if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } else if (auto oid = std::get_if(&curDir)) { + tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); + if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } + } + + assert(entry); + + addToTree(*pathComponents.rbegin(), + *git_tree_entry_id(entry), + git_tree_entry_filemode(entry)); + } + Hash sync() override { updateBuilders({}); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 29d79955480..495916f6283 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -7,7 +7,7 @@ namespace nix { namespace fetchers { struct PublicKey; } -struct GitFileSystemObjectSink : FileSystemObjectSink +struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { /** * Flush builder and return a final Git hash. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4b5d9d26b79..ab0485566de 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -164,7 +164,9 @@ static const Hash nullRev{HashAlgorithm::SHA1}; struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -190,7 +192,7 @@ struct GitInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } @@ -222,7 +224,9 @@ struct GitInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { for (auto & [name, _] : attrs) if (name == "verifyCommit" @@ -238,7 +242,7 @@ struct GitInputScheme : InputScheme throw BadURL("invalid Git branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); @@ -359,13 +363,13 @@ struct GitInputScheme : InputScheme /* URL of the repo, or its path if isLocal. Never a `file` URL. */ std::string url; - void warnDirty() const + void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { - if (!fetchSettings.allowDirty) + if (!settings.allowDirty) throw Error("Git tree '%s' is dirty", url); - if (fetchSettings.warnDirty) + if (settings.warnDirty) warn("Git tree '%s' is dirty", url); } } @@ -646,7 +650,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -704,7 +708,7 @@ struct GitInputScheme : InputScheme // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -736,7 +740,7 @@ struct GitInputScheme : InputScheme verifyCommit(input, repo); } else { - repoInfo.warnDirty(); + repoInfo.warnDirty(*input.settings); if (repoInfo.workdirInfo.headRev) { input.attrs.insert_or_assign("dirtyRev", diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index eb85be578e6..4cf642fc38f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -31,7 +31,9 @@ struct GitArchiveInputScheme : InputScheme { virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const fetchers::Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != schemeName()) return {}; @@ -90,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (ref && rev) throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); @@ -119,12 +121,14 @@ struct GitArchiveInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const fetchers::Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -187,18 +191,20 @@ struct GitArchiveInputScheme : InputScheme } } - std::optional getAccessToken(const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const { - auto tokens = fetchSettings.accessTokens.get(); + auto tokens = settings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host) const { Headers headers; - auto accessToken = getAccessToken(host); + auto accessToken = getAccessToken(settings, host); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -313,7 +319,7 @@ struct GitArchiveInputScheme : InputScheme Git revision alone, we also require a NAR hash or Git tree hash for locking. */ return input.getRev().has_value() - && (fetchSettings.trustTarballsFromGitForges + && (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value() || getTreeHash(input).has_value()); } @@ -372,7 +378,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -389,7 +395,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -409,7 +415,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) const override { auto host = getHost(input); - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -446,7 +452,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -476,7 +482,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } @@ -484,7 +490,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -516,7 +522,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); std::string refUri; if (ref == "HEAD") { @@ -563,14 +569,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); - Input::fromURL(fmt("git+https://%s/%s/%s", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index ba507863138..2e5cd82c78d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -8,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -41,7 +43,7 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -65,13 +67,15 @@ struct IndirectInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 3a5691e6012..6c5aa82759c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -45,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -68,7 +70,7 @@ struct MercurialInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } std::string_view schemeName() const override @@ -88,7 +90,9 @@ struct MercurialInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { parseURL(getStrAttr(attrs, "url")); @@ -97,7 +101,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -174,10 +178,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!fetchSettings.allowDirty) + if (!input.settings->allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (fetchSettings.warnDirty) + if (input.settings->warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 91426b89b54..aa9f5850c8f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -7,14 +7,16 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("path", url.path); @@ -57,12 +59,14 @@ struct PathInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "path"); maybeGetBoolAttr(attrs, "lock"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 52cbac5e0a0..3c893c8ea33 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,7 +1,7 @@ +#include "fetch-settings.hh" #include "registry.hh" #include "tarball.hh" #include "users.hh" -#include "config-global.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -11,12 +11,13 @@ namespace nix::fetchers { std::shared_ptr Registry::read( + const Settings & settings, const Path & path, RegistryType type) { - auto registry = std::make_shared(type); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) - return std::make_shared(type); + return std::make_shared(settings, type); try { @@ -36,8 +37,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = Input::fromAttrs(jsonToAttrs(i["from"])), - .to = Input::fromAttrs(std::move(toAttrs)), + .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])), + .to = Input::fromAttrs(settings, std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -106,10 +107,10 @@ static Path getSystemRegistryPath() return settings.nixConfDir + "/registry.json"; } -static std::shared_ptr getSystemRegistry() +static std::shared_ptr getSystemRegistry(const Settings & settings) { static auto systemRegistry = - Registry::read(getSystemRegistryPath(), Registry::System); + Registry::read(settings, getSystemRegistryPath(), Registry::System); return systemRegistry; } @@ -118,25 +119,24 @@ Path getUserRegistryPath() return getConfigDir() + "/nix/registry.json"; } -std::shared_ptr getUserRegistry() +std::shared_ptr getUserRegistry(const Settings & settings) { static auto userRegistry = - Registry::read(getUserRegistryPath(), Registry::User); + Registry::read(settings, getUserRegistryPath(), Registry::User); return userRegistry; } -std::shared_ptr getCustomRegistry(const Path & p) +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p) { static auto customRegistry = - Registry::read(p, Registry::Custom); + Registry::read(settings, p, Registry::Custom); return customRegistry; } -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(const Settings & settings) { + static auto flagRegistry = + std::make_shared(settings, Registry::Flag); return flagRegistry; } @@ -145,30 +145,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs) { - flagRegistry->add(from, to, extraAttrs); + getFlagRegistry(*from.settings)->add(from, to, extraAttrs); } -struct RegistrySettings : Config -{ - Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - R"( - Path or URI of the global flake registry. - - When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; -}; - -RegistrySettings registrySettings; - -static GlobalConfig::Register rRegistrySettings(®istrySettings); - -static std::shared_ptr getGlobalRegistry(ref store) +static std::shared_ptr getGlobalRegistry(const Settings & settings, ref store) { static auto reg = [&]() { - auto path = registrySettings.flakeRegistry.get(); + auto path = settings.flakeRegistry.get(); if (path == "") { - return std::make_shared(Registry::Global); // empty registry + return std::make_shared(settings, Registry::Global); // empty registry } if (!hasPrefix(path, "/")) { @@ -178,19 +163,19 @@ static std::shared_ptr getGlobalRegistry(ref store) path = store->toRealPath(storePath); } - return Registry::read(path, Registry::Global); + return Registry::read(settings, path, Registry::Global); }(); return reg; } -Registries getRegistries(ref store) +Registries getRegistries(const Settings & settings, ref store) { Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getSystemRegistry()); - registries.push_back(getGlobalRegistry(store)); + registries.push_back(getFlagRegistry(settings)); + registries.push_back(getUserRegistry(settings)); + registries.push_back(getSystemRegistry(settings)); + registries.push_back(getGlobalRegistry(settings, store)); return registries; } @@ -207,7 +192,7 @@ std::pair lookupInRegistries( n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(store)) { + for (auto & registry : getRegistries(*input.settings, store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index f57ab1e6b59..0d68ac395e9 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -10,6 +10,8 @@ namespace nix::fetchers { struct Registry { + const Settings & settings; + enum RegistryType { Flag = 0, User = 1, @@ -29,11 +31,13 @@ struct Registry std::vector entries; - Registry(RegistryType type) - : type(type) + Registry(const Settings & settings, RegistryType type) + : settings{settings} + , type{type} { } static std::shared_ptr read( + const Settings & settings, const Path & path, RegistryType type); void write(const Path & path); @@ -48,13 +52,13 @@ struct Registry typedef std::vector> Registries; -std::shared_ptr getUserRegistry(); +std::shared_ptr getUserRegistry(const Settings & settings); -std::shared_ptr getCustomRegistry(const Path & p); +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p); Path getUserRegistryPath(); -Registries getRegistries(ref store); +Registries getRegistries(const Settings & settings, ref store); void overrideRegistry( const Input & from, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa8ff652f07..55db3eafbc1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -2,12 +2,10 @@ #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" -#include "globals.hh" #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" #include "types.hh" -#include "split.hh" #include "store-path-accessor.hh" #include "store-api.hh" #include "git-utils.hh" @@ -214,12 +212,14 @@ struct CurlInputScheme : InputScheme static const std::set specialParams; - std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & _url, bool requireTree) const override { if (!isValidURL(_url, requireTree)) return std::nullopt; - Input input; + Input input{settings}; auto url = _url; @@ -267,9 +267,11 @@ struct CurlInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { - Input input; + Input input{settings}; input.attrs = attrs; //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); @@ -349,7 +351,7 @@ struct TarballInputScheme : CurlInputScheme result.accessor->setPathDisplay("«" + input.to_string() + "»"); if (result.immutableUrl) { - auto immutableInput = Input::fromURL(*result.immutableUrl); + auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl); // FIXME: would be nice to support arbitrary flakerefs // here, e.g. git flakes. if (immutableInput.getType() != "tarball") diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index ba0dfd6230a..d9bdd123d58 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -1,11 +1,12 @@ #pragma once -#include "types.hh" -#include "path.hh" -#include "hash.hh" - #include +#include "hash.hh" +#include "path.hh" +#include "ref.hh" +#include "types.hh" + namespace nix { class Store; struct SourceAccessor; diff --git a/src/libflake/flake-settings.cc b/src/libflake/flake-settings.cc deleted file mode 100644 index ba97e0ce723..00000000000 --- a/src/libflake/flake-settings.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "flake-settings.hh" -#include "config-global.hh" - -namespace nix { - -FlakeSettings::FlakeSettings() {} - -FlakeSettings flakeSettings; - -static GlobalConfig::Register rFlakeSettings(&flakeSettings); - -} diff --git a/src/libflake/flake/config.cc b/src/libflake/flake/config.cc index 49859535939..e526cdddfa3 100644 --- a/src/libflake/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,6 @@ #include "users.hh" #include "config-global.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "flake.hh" #include @@ -30,7 +30,7 @@ static void writeTrustedList(const TrustedList & trustedList) writeFile(path, nlohmann::json(trustedList).dump()); } -void ConfigFile::apply() +void ConfigFile::apply(const Settings & flakeSettings) { std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; @@ -47,11 +47,11 @@ void ConfigFile::apply() else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil + valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil else assert(false); - if (!whitelist.count(baseName) && !nix::flakeSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 4136e6f406b..a826cb591fc 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -9,7 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" -#include "flake-settings.hh" +#include "flake/settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" #include "patching-source-accessor.hh" @@ -44,7 +44,7 @@ static std::map parseFlakeInputs( static FlakeInput parseFlakeInput( EvalState & state, - const std::string & inputName, + std::string_view inputName, Value * value, const PosIdx pos, const InputPath & lockRootPath, @@ -128,7 +128,7 @@ static FlakeInput parseFlakeInput( if (attrs.count("type")) try { - input.ref = FlakeRef::fromAttrs(attrs); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs); } catch (Error & e) { e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; @@ -138,11 +138,11 @@ static FlakeInput parseFlakeInput( if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, {}, true, input.isFlake, true); + input.ref = parseFlakeRef(state.fetchSettings, *url, {}, true, input.isFlake, true); } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -211,7 +211,7 @@ static Flake readFlake( for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(state.symbols[formal.name]) + .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name])) }); } } @@ -309,37 +309,41 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistri return getFlake(state, originalRef, useRegistries, {}, {}); } -static LockFile readLockFile(const SourcePath & lockFilePath) +static LockFile readLockFile( + const fetchers::Settings & fetchSettings, + const SourcePath & lockFilePath) { return lockFilePath.pathExists() - ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath)) + ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) : LockFile(); } /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { experimentalFeatureSettings.require(Xp::Flakes); - auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, {}, {}); if (lockFlags.applyNixConfig) { - flake.config.apply(); + flake.config.apply(settings); state.store->setOptions(); } try { - if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } auto oldLockFile = readLockFile( + state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -632,7 +636,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? followsPrefix : inputPath, inputFlake.path, false); @@ -702,7 +706,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (auto unlockedInput = newLockFile.isUnlocked()) { - if (fetchSettings.warnDirty) + if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) @@ -731,7 +735,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = flakeSettings.commitLockFileSummary.get(); + cm = settings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", flake.lockFilePath().path.rel(), lockFileExists ? "Update" : "Add"); @@ -822,45 +826,48 @@ void callFlake(EvalState & state, state.callFunction(*vTmp1, vOverrides, vRes, noPos); } -static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) +void initLib(const Settings & settings) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); - - callFlake(state, - lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && flakeSettings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); -} - -static RegisterPrimOp r2({ - .name = "__getFlake", - .args = {"args"}, - .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: - - ```nix - (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix - ``` - - Unless impure evaluation is allowed (`--impure`), the flake reference - must be "locked", e.g. contain a Git revision or content hash. An - example of an unlocked usage is: + auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); + + callFlake(state, + lockFlake(settings, state, flakeRef, + LockFlags { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }), + v); + }; - ```nix - (builtins.getFlake "github:edolstra/dwarffs").rev - ``` - )", - .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, -}); + RegisterPrimOp::primOps->push_back({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, + }); +} static void prim_parseFlakeRef( EvalState & state, @@ -870,7 +877,7 @@ static void prim_parseFlakeRef( { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.parseFlakeRef")); - auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs(); auto binds = state.buildBindings(attrs.size()); for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); @@ -935,7 +942,7 @@ static void prim_flakeRefToString( showType(*attr.value)).debugThrow(); } } - auto flakeRef = FlakeRef::fromAttrs(attrs); + auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); v.mkString(flakeRef.to_string()); } diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 4268081e9ae..69fc6f115df 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -12,6 +12,16 @@ class EvalState; namespace flake { +struct Settings; + +/** + * Initialize `libnixflake` + * + * So far, this registers the `builtins.getFlake` primop, which depends + * on the choice of `flake:Settings`. + */ +void initLib(const Settings & settings); + struct FlakeInput; typedef std::map FlakeInputs; @@ -58,7 +68,7 @@ struct ConfigFile std::map settings; - void apply(); + void apply(const Settings & settings); }; /** @@ -195,6 +205,7 @@ struct LockFlags }; LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 95247b67a30..c319c0e338b 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -36,11 +36,6 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -bool FlakeRef::operator ==(const FlakeRef & other) const -{ - return input == other.input && subdir == other.subdir; -} - FlakeRef FlakeRef::resolve(ref store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); @@ -48,29 +43,33 @@ FlakeRef FlakeRef::resolve(ref store) const } FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake, bool allowRelative) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; } std::optional maybeParseFlakeRef( - const std::string & url, const std::optional & baseDir) + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir) { try { - return parseFlakeRef(url, baseDir); + return parseFlakeRef(fetchSettings, url, baseDir); } catch (Error &) { return {}; } } static std::pair fromParsedURL( + const fetchers::Settings & fetchSettings, ParsedURL && parsedURL, bool isFlake) { @@ -80,10 +79,11 @@ static std::pair fromParsedURL( std::string fragment; std::swap(fragment, parsedURL.fragment); - return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), dir), fragment); + return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment); }; std::pair parsePathFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -181,7 +181,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return fromParsedURL(std::move(parsedURL), isFlake); + return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -194,21 +194,25 @@ std::pair parsePathFlakeRefWithFragment( throw BadURL("flake reference '%s' is not an absolute path", url); } - return fromParsedURL({ - .url = path, // FIXME - .base = path, - .scheme = "path", - .authority = "", - .path = path, - .query = query, - .fragment = fragment - }, isFlake); + return fromParsedURL( + fetchSettings, + { + .url = path, // FIXME + .base = path, + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, + isFlake); }; /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ -std::optional> parseFlakeIdRef( +static std::optional> parseFlakeIdRef( + const fetchers::Settings & fetchSettings, const std::string & url, bool isFlake ) @@ -230,7 +234,7 @@ std::optional> parseFlakeIdRef( }; return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -238,19 +242,21 @@ std::optional> parseFlakeIdRef( } std::optional> parseURLFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool isFlake ) { try { - return fromParsedURL(parseURL(url), isFlake); + return fromParsedURL(fetchSettings, parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } } std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -261,31 +267,34 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - if (auto res = parseFlakeIdRef(url, isFlake)) { + if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; - } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); } } std::optional> maybeParseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir) { try { - return parseFlakeRefWithFragment(url, baseDir); + return parseFlakeRefWithFragment(fetchSettings, url, baseDir); } catch (Error & e) { return {}; } } -FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +FlakeRef FlakeRef::fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs) { auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } @@ -296,13 +305,16 @@ std::pair, FlakeRef> FlakeRef::lazyFetch(ref store) c } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, + std::string { prefix }, baseDir, allowMissing, isFlake); return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index ad4478923f2..ca6da74325b 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -1,14 +1,12 @@ #pragma once ///@file +#include + #include "types.hh" -#include "hash.hh" #include "fetchers.hh" #include "outputs-spec.hh" -#include -#include - namespace nix { class Store; @@ -48,7 +46,7 @@ struct FlakeRef */ Path subdir; - bool operator==(const FlakeRef & other) const; + bool operator ==(const FlakeRef & other) const = default; FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) @@ -61,7 +59,9 @@ struct FlakeRef FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + static FlakeRef fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs); std::pair, FlakeRef> lazyFetch(ref store) const; }; @@ -72,6 +72,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -82,12 +83,15 @@ FlakeRef parseFlakeRef( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional maybeParseFlake( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -98,12 +102,15 @@ std::pair parseFlakeRefWithFragment( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index a3c4e936054..3b11660bba2 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -1,6 +1,7 @@ +#include + #include "lockfile.hh" #include "store-api.hh" -#include "url-parts.hh" #include #include @@ -8,9 +9,12 @@ #include #include +#include "strings.hh" + namespace nix::flake { -FlakeRef getFlakeRef( +static FlakeRef getFlakeRef( + const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) @@ -26,15 +30,17 @@ FlakeRef getFlakeRef( attrs.insert_or_assign(k.first, k.second); } } - return FlakeRef::fromAttrs(attrs); + return FlakeRef::fromAttrs(fetchSettings, attrs); } throw Error("attribute '%s' missing in lock file", attr); } -LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(json, "original", nullptr)) +LockedNode::LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) , parentPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) , patchFiles(json.find("patchFiles") != json.end() ? (std::vector) json["patchFiles"] : std::vector{}) @@ -84,7 +90,9 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return doFind(root, path, visited); } -LockFile::LockFile(std::string_view contents, std::string_view path) +LockFile::LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path) { auto json = nlohmann::json::parse(contents); @@ -113,7 +121,7 @@ LockFile::LockFile(std::string_view contents, std::string_view path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(*jsonNode2); + auto input = make_ref(fetchSettings, *jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } @@ -249,11 +257,6 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON().first == other.toJSON().first; } -bool LockFile::operator !=(const LockFile & other) const -{ - return !(*this == other); -} - InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index 41a3575607a..2563de0e473 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -53,7 +53,9 @@ struct LockedNode : Node : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake), parentPath(parentPath), patchFiles(std::move(patchFiles)) { } - LockedNode(const nlohmann::json & json); + LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json); }; struct LockFile @@ -61,7 +63,9 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(std::string_view contents, std::string_view path); + LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; @@ -76,9 +80,6 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 - // that is still the default on aarch64-linux) - bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); diff --git a/src/libflake/flake/settings.cc b/src/libflake/flake/settings.cc new file mode 100644 index 00000000000..6a0294e6229 --- /dev/null +++ b/src/libflake/flake/settings.cc @@ -0,0 +1,7 @@ +#include "flake/settings.hh" + +namespace nix::flake { + +Settings::Settings() {} + +} diff --git a/src/libflake/flake-settings.hh b/src/libflake/flake/settings.hh similarity index 86% rename from src/libflake/flake-settings.hh rename to src/libflake/flake/settings.hh index f97c175e8a3..fee247a7d2f 100644 --- a/src/libflake/flake-settings.hh +++ b/src/libflake/flake/settings.hh @@ -10,11 +10,11 @@ #include -namespace nix { +namespace nix::flake { -struct FlakeSettings : public Config +struct Settings : public Config { - FlakeSettings(); + Settings(); Setting useRegistries{ this, @@ -47,7 +47,4 @@ struct FlakeSettings : public Config Xp::Flakes}; }; -// TODO: don't use a global variable. -extern FlakeSettings flakeSettings; - } diff --git a/src/libflake/meson.build b/src/libflake/meson.build index d3c3d3079cf..38d70c678a5 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -42,21 +42,21 @@ add_project_arguments( subdir('build-utils-meson/diagnostics') sources = files( - 'flake-settings.cc', 'flake/config.cc', 'flake/flake.cc', 'flake/flakeref.cc', - 'flake/url-name.cc', 'flake/lockfile.cc', + 'flake/settings.cc', + 'flake/url-name.cc', ) include_dirs = [include_directories('.')] headers = files( - 'flake-settings.hh', 'flake/flake.hh', 'flake/flakeref.hh', 'flake/lockfile.hh', + 'flake/settings.hh', 'flake/url-name.hh', ) diff --git a/src/libmain-c/.version b/src/libmain-c/.version new file mode 120000 index 00000000000..b7badcd0cc8 --- /dev/null +++ b/src/libmain-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson new file mode 120000 index 00000000000..5fff21bab55 --- /dev/null +++ b/src/libmain-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build new file mode 100644 index 00000000000..1d6b2f959ff --- /dev/null +++ b/src/libmain-c/meson.build @@ -0,0 +1,84 @@ +project('nix-main-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-main'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-main.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_main.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_main.h', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixmainc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc new file mode 100644 index 00000000000..692d53f47e0 --- /dev/null +++ b/src/libmain-c/nix_api_main.cc @@ -0,0 +1,16 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "plugin.hh" + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h new file mode 100644 index 00000000000..3957b992fd3 --- /dev/null +++ b/src/libmain-c/nix_api_main.h @@ -0,0 +1,40 @@ +#ifndef NIX_API_MAIN_H +#define NIX_API_MAIN_H +/** + * @defgroup libmain libmain + * @brief C bindings for nix libmain + * + * libmain has misc utilities for CLI commands + * @{ + */ +/** @file + * @brief Main entry for the libmain C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_MAIN_H diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix new file mode 100644 index 00000000000..478e34a856a --- /dev/null +++ b/src/libmain-c/package.nix @@ -0,0 +1,83 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store +, nix-store-c +, nix-main + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + nix-store-c + nix-main + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index a94845ab8f3..768b2177c94 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -5,6 +5,7 @@ #include "logging.hh" #include "loggers.hh" #include "util.hh" +#include "plugin.hh" namespace nix { diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 859ce22f8ba..fe6133596a9 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -64,6 +64,7 @@ subdir('build-utils-meson/diagnostics') sources = files( 'common-args.cc', 'loggers.cc', + 'plugin.cc', 'progress-bar.cc', 'shared.cc', ) @@ -79,6 +80,7 @@ include_dirs = [include_directories('.')] headers = [config_h] + files( 'common-args.hh', 'loggers.hh', + 'plugin.hh', 'progress-bar.hh', 'shared.hh', ) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc new file mode 100644 index 00000000000..ccfd7f9003a --- /dev/null +++ b/src/libmain/plugin.cc @@ -0,0 +1,119 @@ +#ifndef _WIN32 +# include +#endif + +#include + +#include "config-global.hh" +#include "signals.hh" + +namespace nix { + +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting( + Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + Paths parse(const std::string & str) const override; +}; + +Paths PluginFilesSetting::parse(const std::string & str) const +{ + if (pluginsLoaded) + throw UsageError( + "plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + return BaseSetting::parse(str); +} + +struct PluginSettings : Config +{ + PluginFilesSetting pluginFiles{ + this, + {}, + "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Warning! These APIs are inherently unstable and may change from + release to release. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; +}; + +static PluginSettings pluginSettings; + +static GlobalConfig::Register rPluginSettings(&pluginSettings); + +void initPlugins() +{ + assert(!pluginSettings.pluginFiles.pluginsLoaded); + for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { + std::vector pluginFiles; + try { + auto ents = std::filesystem::directory_iterator{pluginFile}; + for (const auto & ent : ents) { + checkInterrupt(); + pluginFiles.emplace_back(ent.path()); + } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() != std::errc::not_a_directory) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + checkInterrupt(); + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows + void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)()) dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + pluginSettings.pluginFiles.pluginsLoaded = true; +} + +} diff --git a/src/libmain/plugin.hh b/src/libmain/plugin.hh new file mode 100644 index 00000000000..4221c1b1713 --- /dev/null +++ b/src/libmain/plugin.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +namespace nix { + +/** + * This should be called after settings are initialized, but before + * anything else + */ +void initPlugins(); + +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index fc55fe3f1b2..a224f8d92a4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -8,7 +8,6 @@ #include "signals.hh" #include -#include #include #include @@ -23,6 +22,8 @@ #include +#include "exit.hh" +#include "strings.hh" namespace nix { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index aa44e1321fa..712b404d3e1 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -8,13 +8,9 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" -#include "exit.hh" #include -#include - - namespace nix { int handleExceptions(const std::string & programName, std::function fun); diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4fe25c7d4ee..79841ca49a1 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -29,16 +29,6 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context) NIXC_CATCH_ERRS } -nix_err nix_init_plugins(nix_c_context * context) -{ - if (context) - context->last_err_code = NIX_OK; - try { - nix::initPlugins(); - } - NIXC_CATCH_ERRS -} - Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index d3cb8fab84e..4b213445778 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -42,17 +42,6 @@ nix_err nix_libstore_init(nix_c_context * context); */ nix_err nix_libstore_init_no_load_config(nix_c_context * context); -/** - * @brief Loads the plugins specified in Nix's plugin-files setting. - * - * Call this once, after calling your desired init functions and setting - * relevant settings. - * - * @param[out] context Optional, stores error information - * @return NIX_OK if the initialization was successful, an error code otherwise. - */ -nix_err nix_init_plugins(nix_c_context * context); - /** * @brief Open a nix store. * diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 18f519c5c61..96cbfd62fff 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -2,17 +2,7 @@ namespace nix { -GENERATE_CMP_EXT( - , - BuildResult, - me->status, - me->errorMsg, - me->timesBuilt, - me->isNonDeterministic, - me->builtOutputs, - me->startTime, - me->stopTime, - me->cpuUser, - me->cpuSystem); +bool BuildResult::operator==(const BuildResult &) const noexcept = default; +std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 3636ad3a4c3..8c66cfeb353 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,7 +3,6 @@ #include "realisation.hh" #include "derived-path.hh" -#include "comparator.hh" #include #include @@ -101,7 +100,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; - DECLARE_CMP(BuildResult); + bool operator ==(const BuildResult &) const noexcept; + std::strong_ordering operator <=>(const BuildResult &) const noexcept; bool success() { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 64b8495e1bb..b809e3ffe3f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -32,8 +32,18 @@ #include +#include "strings.hh" + namespace nix { +Goal::Co DerivationGoal::init() { + if (useDerivation) { + co_return getDerivation(); + } else { + co_return haveDerivation(); + } +} + DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) @@ -42,7 +52,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -63,7 +72,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -107,13 +115,9 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, std::move(ex)); -} - - -void DerivationGoal::work() -{ - (this->*state)(); + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -137,7 +141,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() +Goal::Co DerivationGoal::getDerivation() { trace("init"); @@ -145,23 +149,22 @@ void DerivationGoal::getDerivation() exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; + co_return loadDerivation(); } addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - state = &DerivationGoal::loadDerivation; + co_await Suspend{}; + co_return loadDerivation(); } -void DerivationGoal::loadDerivation() +Goal::Co DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); } /* `drvPath' should already be a root, but let's be on the safe @@ -183,11 +186,11 @@ void DerivationGoal::loadDerivation() } assert(drv); - haveDerivation(); + co_return haveDerivation(); } -void DerivationGoal::haveDerivation() +Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); @@ -215,8 +218,7 @@ void DerivationGoal::haveDerivation() }); } - gaveUpOnSubstitution(); - return; + co_return gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -238,8 +240,7 @@ void DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -266,24 +267,21 @@ void DerivationGoal::haveDerivation() } } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return outputsSubstitutionTried(); } -void DerivationGoal::outputsSubstitutionTried() +Goal::Co DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, {}, + co_return done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); - return; } /* If the substitutes form an incomplete closure, then we should @@ -317,32 +315,29 @@ void DerivationGoal::outputsSubstitutionTried() if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - haveDerivation(); - return; + co_return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - done(BuildResult::Substituted, std::move(validOutputs)); - return; + co_return done(BuildResult::Substituted, std::move(validOutputs)); } if (buildMode == bmRepair && allValid) { - repairClosure(); - return; + co_return repairClosure(); } if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); + co_return gaveUpOnSubstitution(); } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() +Goal::Co DerivationGoal::gaveUpOnSubstitution() { /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -403,14 +398,12 @@ void DerivationGoal::gaveUpOnSubstitution() addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return inputsRealised(); } -void DerivationGoal::repairClosure() +Goal::Co DerivationGoal::repairClosure() { assert(!drv->type().isImpure()); @@ -464,41 +457,39 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid, assertPathValidity()); - return; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); + } else { + co_await Suspend{}; + co_return closureRepaired(); } - - state = &DerivationGoal::closureRepaired; } -void DerivationGoal::closureRepaired() +Goal::Co DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid, assertPathValidity()); + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -void DerivationGoal::inputsRealised() +Goal::Co DerivationGoal::inputsRealised() { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, {}, Error( + co_return done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); - return; } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - haveDerivation(); - return; + co_return haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -564,8 +555,8 @@ void DerivationGoal::inputsRealised() pathResolved, wantedOutputs, buildMode); addWaitee(resolvedDrvGoal); - state = &DerivationGoal::resolvedFinished; - return; + co_await Suspend{}; + co_return resolvedFinished(); } std::function::ChildNode &)> accumInputPaths; @@ -629,8 +620,9 @@ void DerivationGoal::inputsRealised() /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryToBuild(); } void DerivationGoal::started() @@ -655,7 +647,7 @@ void DerivationGoal::started() worker.updateProgress(); } -void DerivationGoal::tryToBuild() +Goal::Co DerivationGoal::tryToBuild() { trace("trying to build"); @@ -691,7 +683,8 @@ void DerivationGoal::tryToBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryToBuild(); } actLock.reset(); @@ -708,8 +701,7 @@ void DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -735,9 +727,9 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; started(); - return; + co_await Suspend{}; + co_return buildDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ @@ -746,7 +738,8 @@ void DerivationGoal::tryToBuild() fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ break; @@ -755,11 +748,12 @@ void DerivationGoal::tryToBuild() actLock.reset(); - state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryLocalBuild(); } -void DerivationGoal::tryLocalBuild() { +Goal::Co DerivationGoal::tryLocalBuild() { throw Error( R"( Unable to build with a primary store that isn't a local store; @@ -936,7 +930,7 @@ void runPostBuildHook( }); } -void DerivationGoal::buildDone() +Goal::Co DerivationGoal::buildDone() { trace("build done"); @@ -1031,7 +1025,7 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); - done(BuildResult::Built, std::move(builtOutputs)); + co_return done(BuildResult::Built, std::move(builtOutputs)); } catch (BuildError & e) { outputLocks.unlock(); @@ -1056,12 +1050,11 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, std::move(e)); - return; + co_return done(st, {}, std::move(e)); } } -void DerivationGoal::resolvedFinished() +Goal::Co DerivationGoal::resolvedFinished() { trace("resolved derivation finished"); @@ -1129,7 +1122,7 @@ void DerivationGoal::resolvedFinished() if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - done(status, std::move(builtOutputs)); + co_return done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1323,7 +1316,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); - done( + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done( BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); @@ -1529,7 +1524,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -void DerivationGoal::done( +Goal::Done DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) @@ -1566,7 +1561,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } @@ -1574,7 +1569,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (!useDerivation) return; + if (!useDerivation || !drv) return; auto & fullDrv = *dynamic_cast(drv.get()); auto * dg = dynamic_cast(&*waitee); diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 04f13aedd17..ad3d9ca2acf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -194,9 +194,6 @@ struct DerivationGoal : public Goal */ std::optional derivationType; - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - BuildMode buildMode; std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; @@ -227,8 +224,6 @@ struct DerivationGoal : public Goal std::string key() override; - void work() override; - /** * Add wanted outputs to an already existing derivation goal. */ @@ -237,18 +232,19 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - virtual void tryLocalBuild(); - void buildDone(); + Co init() override; + Co getDerivation(); + Co loadDerivation(); + Co haveDerivation(); + Co outputsSubstitutionTried(); + Co gaveUpOnSubstitution(); + Co closureRepaired(); + Co inputsRealised(); + Co tryToBuild(); + virtual Co tryLocalBuild(); + Co buildDone(); - void resolvedFinished(); + Co resolvedFinished(); /** * Is the build hook willing to perform the build? @@ -329,11 +325,11 @@ struct DerivationGoal : public Goal */ virtual void killChild(); - void repairClosure(); + Co repairClosure(); void started(); - void done( + Done done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 13a07e4ea38..02284d93c1a 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -14,146 +14,135 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -void DrvOutputSubstitutionGoal::init() +Goal::Co DrvOutputSubstitutionGoal::init() { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - amDone(ecSuccess); - return; + co_return amDone(ecSuccess); } - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); -} - -void DrvOutputSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + auto outPipe = std::make_shared(); + #ifndef _WIN32 + outPipe->create(); + #else + outPipe->createAsyncPipe(worker.ioport.get()); + #endif + + auto promise = std::make_shared>>(); + + sub->queryRealisation( + id, + { [outPipe(outPipe), promise(promise)](std::future> res) { + try { + Finally updateStats([&]() { outPipe->writeSide.close(); }); + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), { + #ifndef _WIN32 + outPipe->readSide.get() + #else + &outPipe + #endif + }, true, false); + + co_await Suspend{}; + + worker.childTerminated(this); + + /* + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; + + try { + outputInfo = promise->get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; } - return; - } - - sub = subs.front(); - subs.pop_front(); - - // FIXME: Make async - // outputInfo = sub->queryRealisation(id); - - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - downloadState = std::make_shared(); -#ifndef _WIN32 - downloadState->outPipe.create(); -#else - downloadState->outPipe.createAsyncPipe(worker.ioport.get()); -#endif - - sub->queryRealisation( - id, - { [downloadState(downloadState)](std::future> res) { - try { - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); - downloadState->promise.set_value(res.get()); - } catch (...) { - downloadState->promise.set_exception(std::current_exception()); + if (!outputInfo) continue; + + bool failed = false; + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + failed = true; + break; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); } - } }); - - worker.childStarted(shared_from_this(), { -#ifndef _WIN32 - downloadState->outPipe.readSide.get() -#else - &downloadState->outPipe -#endif - }, true, false); - - state = &DrvOutputSubstitutionGoal::realisationFetched; -} + } -void DrvOutputSubstitutionGoal::realisationFetched() -{ - worker.childTerminated(this); + if (failed) continue; - try { - outputInfo = downloadState->promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - substituterFailed = true; + co_return realisationFetched(outputInfo, sub); } - if (!outputInfo) { - return tryNext(); - } + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - tryNext(); - return; - } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); - } + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); } + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); +} + +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (waitees.empty()) outPathValid(); - else state = &DrvOutputSubstitutionGoal::outPathValid; -} + if (!waitees.empty()) co_await Suspend{}; -void DrvOutputSubstitutionGoal::outPathValid() -{ - assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; + co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); } worker.store.registerDrvOutput(*outputInfo); - finished(); -} -void DrvOutputSubstitutionGoal::finished() -{ trace("finished"); - amDone(ecSuccess); + co_return amDone(ecSuccess); } std::string DrvOutputSubstitutionGoal::key() @@ -163,14 +152,9 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -void DrvOutputSubstitutionGoal::work() -{ - (this->*state)(); -} - void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 6967ca84ff8..80705492662 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -27,52 +27,19 @@ class DrvOutputSubstitutionGoal : public Goal { */ DrvOutput id; - /** - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - struct DownloadState - { - MuxablePipe outPipe; - std::promise> promise; - }; - - std::shared_ptr downloadState; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - public: DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); typedef void (DrvOutputSubstitutionGoal::*GoalState)(); GoalState state; - void init(); - void tryNext(); - void realisationFetched(); - void outPathValid(); - void finished(); + Co init() override; + Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); void timedOut(Error && ex) override { abort(); }; std::string key() override; - void work() override; void handleEOF(Descriptor fd) override; JobCategory jobCategory() const override { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 784f618c1a0..4c1373bfaff 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -4,6 +4,7 @@ # include "derivation-goal.hh" #endif #include "local-store.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index f8db9828076..9a16da14555 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -3,6 +3,97 @@ namespace nix { +using Co = nix::Goal::Co; +using promise_type = nix::Goal::promise_type; +using handle_type = nix::Goal::handle_type; +using Suspend = nix::Goal::Suspend; + +Co::Co(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +void Co::operator=(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +Co::~Co() { + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } +} + +Co promise_type::get_return_object() { + auto handle = handle_type::from_promise(*this); + return Co{handle}; +}; + +std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept { + auto& p = h.promise(); + auto goal = p.goal; + assert(goal); + goal->trace("in final_awaiter"); + auto c = std::move(p.continuation); + + if (c) { + // We still have a continuation, i.e. work to do. + // We assert that the goal is still busy. + assert(goal->exitCode == ecBusy); + assert(goal->top_co); // Goal must have an active coroutine. + assert(goal->top_co->handle == h); // The active coroutine must be us. + assert(p.alive); // We must not have been destructed. + + // we move continuation to the top, + // note: previous top_co is actually h, so by moving into it, + // we're calling the destructor on h, DON'T use h and p after this! + + // We move our continuation into `top_co`, i.e. the marker for the active continuation. + // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore. + // Be careful not to access freed memory! + goal->top_co = std::move(c); + + // We resume `top_co`. + return goal->top_co->handle; + } else { + // We have no continuation, i.e. no more work to do, + // so the goal must not be busy anymore. + assert(goal->exitCode != ecBusy); + + // We reset `top_co` for good measure. + p.goal->top_co = {}; + + // We jump to the noop coroutine, which doesn't do anything and immediately suspends. + // This passes control back to the caller of goal.work(). + return std::noop_coroutine(); + } +} + +void promise_type::return_value(Co&& next) { + goal->trace("return_value(Co&&)"); + // Save old continuation. + auto old_continuation = std::move(continuation); + // We set next as our continuation. + continuation = std::move(next); + // We set next's goal, and thus it must not have one already. + assert(!continuation->handle.promise().goal); + continuation->handle.promise().goal = goal; + // Nor can next have a continuation, as we set it to our old one. + assert(!continuation->handle.promise().continuation); + continuation->handle.promise().continuation = std::move(old_continuation); +} + +std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) { + assert(handle); // we must be a valid coroutine + auto& p = handle.promise(); + assert(!p.continuation); // we must have no continuation + assert(!p.goal); // we must not have a goal yet + auto goal = caller.promise().goal; + assert(goal); + p.goal = goal; + p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller) + goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this! + return p.goal->top_co->handle; // we execute ourselves +} bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { std::string s1 = a->key(); @@ -75,10 +166,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } } - -void Goal::amDone(ExitCode result, std::optional ex) +Goal::Done Goal::amDone(ExitCode result, std::optional ex) { trace("done"); + assert(top_co); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; @@ -98,6 +189,13 @@ void Goal::amDone(ExitCode result, std::optional ex) worker.removeGoal(shared_from_this()); cleanup(); + + // We drop the continuation. + // In `final_awaiter` this will signal that there is no more work to be done. + top_co->handle.promise().continuation = {}; + + // won't return to caller because of logic in final_awaiter + return Done{}; } @@ -106,4 +204,16 @@ void Goal::trace(std::string_view s) debug("%1%: %2%", name, s); } +void Goal::work() +{ + assert(top_co); + assert(top_co->handle); + assert(top_co->handle.promise().alive); + top_co->handle.resume(); + // We either should be in a state where we can be work()-ed again, + // or we should be done. + assert(top_co || exitCode != ecBusy); +} + + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0d9b828e1d6..162c392d066 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,10 +1,11 @@ #pragma once ///@file -#include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include + namespace nix { /** @@ -103,9 +104,263 @@ protected: * Build result. */ BuildResult buildResult; - public: + /** + * Suspend our goal and wait until we get @ref work()-ed again. + * `co_await`-able by @ref Co. + */ + struct Suspend {}; + + /** + * Return from the current coroutine and suspend our goal + * if we're not busy anymore, or jump to the next coroutine + * set to be executed/resumed. + */ + struct Return {}; + + /** + * `co_return`-ing this will end the goal. + * If you're not inside a coroutine, you can safely discard this. + */ + struct [[nodiscard]] Done { + private: + Done(){} + + friend Goal; + }; + + // forward declaration of promise_type, see below + struct promise_type; + + /** + * Handle to coroutine using @ref Co and @ref promise_type. + */ + using handle_type = std::coroutine_handle; + + /** + * C++20 coroutine wrapper for use in goal logic. + * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co). + * + * @ref Co is meant to be used by methods of subclasses of @ref Goal. + * The main functionality provided by `Co` is + * - `co_await Suspend{}`: Suspends the goal. + * - `co_await f()`: Waits until `f()` finishes. + * - `co_return f()`: Tail-calls `f()`. + * - `co_return Return{}`: Ends coroutine. + * + * The idea is that you implement the goal logic using coroutines, + * and do the core thing a goal can do, suspension, when you have + * children you're waiting for. + * Coroutines allow you to resume the work cleanly. + * + * @note Brief explanation of C++20 coroutines: + * When you `Co f()`, a `std::coroutine_handle` is created, + * alongside its @ref promise_type. + * There are suspension points at the beginning of the coroutine, + * at every `co_await`, and at the final (possibly implicit) `co_return`. + * Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`. + * Suspension points are implemented by passing a struct to the compiler + * that implements `await_sus`pend. + * `await_suspend` can either say "cancel suspension", in which case execution resumes, + * "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()` + * or the place where the coroutine function is initially executed in the case of the initial + * suspension, or `await_suspend` can specify another coroutine to jump to, which is + * how tail calls are implemented. + * + * @note Resources: + * - https://lewissbaker.github.io/ + * - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/ + * - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html + * + * @todo Allocate explicitly on stack since HALO thing doesn't really work, + * specifically, there's no way to uphold the requirements when trying to do + * tail-calls without using a trampoline AFAICT. + * + * @todo Support returning data natively + */ + struct [[nodiscard]] Co { + /** + * The underlying handle. + */ + handle_type handle; + + explicit Co(handle_type handle) : handle(handle) {}; + void operator=(Co&&); + Co(Co&& rhs); + ~Co(); + + bool await_ready() { return false; }; + /** + * When we `co_await` another @ref Co-returning coroutine, + * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle). + * To make sure we return to the original coroutine, we set it as the continuation of our + * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it. + * + * To explain in more understandable terms: + * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of + * the _called_ function, and C++ automatically passes the caller in. + * + * `goal` field of @ref promise_type is also set here by copying it from the caller. + */ + std::coroutine_handle<> await_suspend(handle_type handle); + void await_resume() {}; + }; + + /** + * Used on initial suspend, does the same as @ref std::suspend_always, + * but asserts that everything has been set correctly. + */ + struct InitialSuspend { + /** + * Handle of coroutine that does the + * initial suspend + */ + handle_type handle; + + bool await_ready() { return false; }; + void await_suspend(handle_type handle_) { + handle = handle_; + } + void await_resume() { + assert(handle); + assert(handle.promise().goal); // goal must be set + assert(handle.promise().goal->top_co); // top_co of goal must be set + assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us + } + }; + + /** + * Promise type for coroutines defined using @ref Co. + * Attached to coroutine handle. + */ + struct promise_type { + /** + * Either this is who called us, or it is who we will tail-call. + * It is what we "jump" to once we are done. + */ + std::optional continuation; + + /** + * The goal that we're a part of. + * Set either in @ref Co::await_suspend or in constructor of @ref Goal. + */ + Goal* goal = nullptr; + + /** + * Is set to false when destructed to ensure we don't use a + * destructed coroutine by accident + */ + bool alive = true; + + /** + * The awaiter used by @ref final_suspend. + */ + struct final_awaiter { + bool await_ready() noexcept { return false; }; + /** + * Here we execute our continuation, by passing it back to the caller. + * C++ compiler will create code that takes that and executes it promptly. + * `h` is the handle for the coroutine that is finishing execution, + * thus it must be destroyed. + */ + std::coroutine_handle<> await_suspend(handle_type h) noexcept; + void await_resume() noexcept { assert(false); }; + }; + + /** + * Called by compiler generated code to construct the @ref Co + * that is returned from a @ref Co-returning coroutine. + */ + Co get_return_object(); + + /** + * Called by compiler generated code before body of coroutine. + * We use this opportunity to set the @ref goal field + * and `top_co` field of @ref Goal. + */ + InitialSuspend initial_suspend() { return {}; }; + + /** + * Called on `co_return`. Creates @ref final_awaiter which + * either jumps to continuation or suspends goal. + */ + final_awaiter final_suspend() noexcept { return {}; }; + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Return) {} + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Done) {} + + /** + * When "returning" another coroutine, what happens is that + * we set it as our own continuation, thus once the final suspend + * happens, we transfer control to it. + * The original continuation we had is set as the continuation + * of the coroutine passed in. + * @ref final_suspend is called after this, and @ref final_awaiter will + * pass control off to @ref continuation. + * + * If we already have a continuation, that continuation is set as + * the continuation of the new continuation. Thus, the continuation + * passed to @ref return_value must not have a continuation set. + */ + void return_value(Co&&); + + /** + * If an exception is thrown inside a coroutine, + * we re-throw it in the context of the "resumer" of the continuation. + */ + void unhandled_exception() { throw; }; + + /** + * Allows awaiting a @ref Co. + */ + Co&& await_transform(Co&& co) { return static_cast(co); } + + /** + * Allows awaiting a @ref Suspend. + * Always suspends. + */ + std::suspend_always await_transform(Suspend) { return {}; }; + }; + + /** + * The coroutine being currently executed. + * MUST be updated when switching the coroutine being executed. + * This is used both for memory management and to resume the last + * coroutine executed. + * Destroying this should destroy all coroutines created for this goal. + */ + std::optional top_co; + + /** + * The entry point for the goal + */ + virtual Co init() = 0; + + /** + * Wrapper around @ref init since virtual functions + * can't be used in constructors. + */ + inline Co init_wrapper(); + + /** + * Signals that the goal is done. + * `co_return` the result. If you're not inside a coroutine, you can ignore + * the return value safely. + */ + Done amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } + /** * Project a `BuildResult` with just the information that pertains * to the given request. @@ -124,15 +379,20 @@ public: std::optional ex; Goal(Worker & worker, DerivedPath path) - : worker(worker) - { } + : worker(worker), top_co(init_wrapper()) + { + // top_co shouldn't have a goal already, should be nullptr. + assert(!top_co->handle.promise().goal); + // we set it such that top_co can pass it down to its subcoroutines. + top_co->handle.promise().goal = this; + } virtual ~Goal() { trace("goal destroyed"); } - virtual void work() = 0; + void work(); void addWaitee(GoalPtr waitee); @@ -164,10 +424,6 @@ public: virtual std::string key() = 0; - void amDone(ExitCode result, std::optional ex = {}); - - virtual void cleanup() { } - /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory @@ -178,3 +434,12 @@ public: void addToWeakGoals(WeakGoals & goals, GoalPtr p); } + +template +struct std::coroutine_traits { + using promise_type = nix::Goal::promise_type; +}; + +nix::Goal::Co nix::Goal::init_wrapper() { + co_return init(); +} diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 0be3d1e8d98..7deeb47487d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,7 @@ #include "nar-info.hh" #include "finally.hh" #include "signals.hh" +#include namespace nix { @@ -12,7 +13,6 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); @@ -25,7 +25,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done( +Goal::Done PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional errorMsg) @@ -35,17 +35,11 @@ void PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - amDone(result); + return amDone(result); } -void PathSubstitutionGoal::work() -{ - (this->*state)(); -} - - -void PathSubstitutionGoal::init() +Goal::Co PathSubstitutionGoal::init() { trace("init"); @@ -53,152 +47,135 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - done(ecSuccess, BuildResult::AlreadyValid); - return; + co_return done(ecSuccess, BuildResult::AlreadyValid); } if (settings.readOnlyMode) throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + bool substituterFailed = false; -void PathSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); + for (auto sub : subs) { + trace("trying next substituter"); - cleanup(); + cleanup(); - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - done( - substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; - return; - } + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; + if (ca) { + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + continue; } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; + + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + } catch (InvalidPath &) { + continue; + } catch (SubstituterDisabled & e) { + if (settings.tryFallback) continue; + else throw e; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + continue; + } else throw e; } - throw; - } - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + continue; + } } - } - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; - worker.updateProgress(); + worker.updateProgress(); - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) - { - warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", - worker.store.printStorePath(storePath), sub->getUri()); - tryNext(); - return; + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + { + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); + continue; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); + + if (!waitees.empty()) co_await Suspend{}; + + // FIXME: consider returning boolean instead of passing in reference + bool out = false; // is mutated by tryToRun + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + substituterFailed = substituterFailed || out; } - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + /* None left. Terminate this goal and let someone else deal + with it. */ - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &PathSubstitutionGoal::referencesValid; + worker.failedSubstitutions++; + worker.updateProgress(); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } -void PathSubstitutionGoal::referencesValid() +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) { trace("all references realised"); if (nrFailed > 0) { - done( + co_return done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, BuildResult::DependencyFailed, fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); - return; } for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); -} - + co_await Suspend{}; -void PathSubstitutionGoal::tryToRun() -{ trace("trying to run"); /* Make sure that we are allowed to start a substitution. Note that even @@ -206,10 +183,10 @@ void PathSubstitutionGoal::tryToRun() prevents infinite waiting. */ if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { worker.waitForBuildSlot(shared_from_this()); - return; + co_await Suspend{}; } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); #ifndef _WIN32 @@ -218,9 +195,9 @@ void PathSubstitutionGoal::tryToRun() outPipe.createAsyncPipe(worker.ioport.get()); #endif - promise = std::promise(); + auto promise = std::promise(); - thr = std::thread([this]() { + thr = std::thread([this, &promise, &subPath, &sub]() { try { ReceiveInterrupts receiveInterrupts; @@ -231,7 +208,7 @@ void PathSubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -247,12 +224,8 @@ void PathSubstitutionGoal::tryToRun() #endif }, true, false); - state = &PathSubstitutionGoal::finished; -} - + co_await Suspend{}; -void PathSubstitutionGoal::finished() -{ trace("substitute finished"); thr.join(); @@ -274,10 +247,7 @@ void PathSubstitutionGoal::finished() substituterFailed = true; } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; + co_return Return{}; } worker.markContentsGood(storePath); @@ -295,23 +265,19 @@ void PathSubstitutionGoal::finished() worker.doneDownloadSize += fileSize; } + assert(maintainExpectedNar); worker.doneNarSize += maintainExpectedNar->delta; maintainExpectedNar.reset(); worker.updateProgress(); - done(ecSuccess, BuildResult::Substituted); -} - - -void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data) -{ + co_return done(ecSuccess, BuildResult::Substituted); } void PathSubstitutionGoal::handleEOF(Descriptor fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1a051fc1fd5..86e4f542382 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,14 +1,16 @@ #pragma once ///@file +#include "worker.hh" #include "store-api.hh" #include "goal.hh" #include "muxable-pipe.hh" +#include +#include +#include namespace nix { -class Worker; - struct PathSubstitutionGoal : public Goal { /** @@ -17,30 +19,9 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. - */ - std::optional subPath; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - - /** - * Path info returned by the substituter's query info operation. + * Whether to try to repair a valid path. */ - std::shared_ptr info; + RepairFlag repair; /** * Pipe for the substituter's standard output. @@ -52,31 +33,15 @@ struct PathSubstitutionGoal : public Goal */ std::thread thr; - std::promise promise; - - /** - * Whether to try to repair a valid path. - */ - RepairFlag repair; - - /** - * Location where we're downloading the substitute. Differs from - * storePath when doing a repair. - */ - Path destPath; - std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (PathSubstitutionGoal::*GoalState)(); - GoalState state; - /** * Content address for recomputing store path */ std::optional ca; - void done( + Done done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); @@ -96,22 +61,18 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work() override; - /** * The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + Co init() override; + Co gotInfo(); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co finished(); /** * Callback used by the worker to write to the log. */ - void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleChildOutput(Descriptor fd, std::string_view data) override {}; void handleEOF(Descriptor fd) override; /* Called by destructor, can't be overridden */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 8a5d6de725f..7fc41b1214a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -337,31 +337,27 @@ void Worker::run(const Goals & _topGoals) /* Wait for input. */ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); - else { - if (awake.empty() && 0U == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error( - R"( - Unable to start any build; - either increase '--max-jobs' or enable remote builds. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); - else - throw Error( - R"( - Unable to start any build; - remote machines may not have all required system features. - - For more information run 'man nix.conf' and search for '/machines'. - )" - ); + else if (awake.empty() && 0U == settings.maxBuildJobs) { + if (getMachines().empty()) + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + else + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. - } - assert(!awake.empty()); - } + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + + } else assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc similarity index 96% rename from src/libstore/ssh-store-config.cc rename to src/libstore/common-ssh-store-config.cc index e81a9487449..05332b9bb5c 100644 --- a/src/libstore/ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -1,6 +1,6 @@ #include -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "ssh.hh" namespace nix { diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh similarity index 100% rename from src/libstore/ssh-store-config.hh rename to src/libstore/common-ssh-store-config.hh diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6cc3b7cd9e7..bb515013a04 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -73,6 +73,7 @@ struct ContentAddressMethod Raw raw; + bool operator ==(const ContentAddressMethod &) const = default; auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -159,6 +160,7 @@ struct ContentAddress */ Hash hash; + bool operator ==(const ContentAddress &) const = default; auto operator <=>(const ContentAddress &) const = default; /** @@ -218,7 +220,9 @@ struct StoreReferences */ size_t size() const; - auto operator <=>(const StoreReferences &) const = default; + bool operator ==(const StoreReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -235,7 +239,9 @@ struct TextInfo */ StorePathSet references; - auto operator <=>(const TextInfo &) const = default; + bool operator ==(const TextInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -255,7 +261,9 @@ struct FixedOutputInfo */ StoreReferences references; - auto operator <=>(const FixedOutputInfo &) const = default; + bool operator ==(const FixedOutputInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -272,7 +280,9 @@ struct ContentAddressWithReferences Raw raw; - auto operator <=>(const ContentAddressWithReferences &) const = default; + bool operator ==(const ContentAddressWithReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 40163a6214d..6533b2f58c4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -246,10 +246,9 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); - } else if (name == settings.pluginFiles.name) { - if (tokenizeString(value) != settings.pluginFiles.get()) - warn("Ignoring the client-specified plugin-files.\n" - "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); + } else if (name == "plugin-files") { + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -270,26 +269,21 @@ struct ClientSettings }; static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, - Source & from, BufferedSink & to, WorkerProto::Op op) + TrustedFlag trusted, RecursiveFlag recursive, + WorkerProto::BasicServerConnection & conn, + WorkerProto::Op op) { - WorkerProto::ReadConn rconn { - .from = from, - .version = clientVersion, - }; - WorkerProto::WriteConn wconn { - .to = to, - .version = clientVersion, - }; + WorkerProto::ReadConn rconn(conn); + WorkerProto::WriteConn wconn(conn); switch (op) { case WorkerProto::Op::IsValidPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); - to << result; + conn.to << result; break; } @@ -297,8 +291,8 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { - substitute = readInt(from) ? Substitute : NoSubstitute; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + substitute = readInt(conn.from) ? Substitute : NoSubstitute; } logger->startWork(); @@ -312,13 +306,13 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::HasSubstitutes: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.count(path) != 0); + conn.to << (res.count(path) != 0); break; } @@ -332,11 +326,11 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathHash: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(HashFormat::Base16, false); + conn.to << hash.to_string(HashFormat::Base16, false); break; } @@ -344,7 +338,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) @@ -361,16 +355,16 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); - to << names; + conn.to << names; break; } case WorkerProto::Op::QueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); @@ -379,37 +373,37 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDeriver: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case WorkerProto::Op::QueryPathFromHashPart: { - auto hashPart = readString(from); + auto hashPart = readString(conn.from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << (path ? store->printStorePath(*path) : ""); + conn.to << (path ? store->printStorePath(*path) : ""); break; } case WorkerProto::Op::AddToStore: { - if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { - auto name = readString(from); - auto camStr = readString(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + auto name = readString(conn.from); + auto camStr = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; - from >> repairBool; + conn.from >> repairBool; auto repair = RepairFlag{repairBool}; logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); - FramedSource source(from); + FramedSource source(conn.from); FileSerialisationMethod dumpMethod; switch (contentAddressMethod.getFileIngestionMethod()) { case FileIngestionMethod::Flat: @@ -440,7 +434,7 @@ static void performOp(TunnelLogger * logger, ref store, bool fixed; uint8_t recursive; std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + conn.from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = recursive @@ -460,11 +454,11 @@ static void performOp(TunnelLogger * logger, ref store, so why all this extra work? We still parse the NAR so that we aren't sending arbitrary data to `saved` unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another + consume the rest of `conn.from` and can't parse another command. (We don't trust `addToStoreFromDump` to not eagerly consume the entire stream it's given, past the length of the Nar. */ - TeeSource savedNARSource(from, saved); + TeeSource savedNARSource(conn.from, saved); NullFileSystemObjectSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); }); @@ -473,20 +467,20 @@ static void performOp(TunnelLogger * logger, ref store, *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); } break; } case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; - from >> repair >> dontCheckSigs; + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); @@ -496,8 +490,8 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddTextToStore: { - std::string suffix = readString(from); - std::string s = readString(from); + std::string suffix = readString(conn.from); + std::string s = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto path = ({ @@ -505,37 +499,37 @@ static void performOp(TunnelLogger * logger, ref store, store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); break; } case WorkerProto::Op::ExportPath: { - auto path = store->parseStorePath(readString(from)); - readInt(from); // obsolete + auto path = store->parseStorePath(readString(conn.from)); + readInt(conn.from); // obsolete logger->startWork(); - TunnelSink sink(to); + TunnelSink sink(conn.to); store->exportPath(path, sink); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::ImportPaths: { logger->startWork(); - TunnelSource source(from, to); + TunnelSource source(conn.from, conn.to); auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; for (auto & i : paths) paths2.push_back(store->printStorePath(i)); - to << paths2; + conn.to << paths2; break; } case WorkerProto::Op::BuildPaths: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" @@ -553,7 +547,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -579,7 +573,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildDerivation: { - auto drvPath = store->parseStorePath(readString(from)); + auto drvPath = store->parseStorePath(readString(conn.from)); BasicDerivation drv; /* * Note: unlike wopEnsurePath, this operation reads a @@ -590,7 +584,7 @@ static void performOp(TunnelLogger * logger, ref store, * it cannot be trusted that its outPath was calculated * correctly. */ - readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); + readDerivation(conn.from, *store, drv, Derivation::nameFromPath(drvPath)); auto buildMode = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); @@ -656,20 +650,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::EnsurePath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::AddTempRoot: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -679,24 +673,24 @@ static void performOp(TunnelLogger * logger, ref store, "you are not privileged to create perm roots\n\n" "hint: you can just do this client-side without special privileges, and probably want to do that instead."); auto storePath = WorkerProto::Serialise::read(*store, rconn); - Path gcRoot = absPath(readString(from)); + Path gcRoot = absPath(readString(conn.from)); logger->startWork(); auto & localFSStore = require(*store); localFSStore.addPermRoot(storePath, gcRoot); logger->stopWork(); - to << gcRoot; + conn.to << gcRoot; break; } case WorkerProto::Op::AddIndirectRoot: { - Path path = absPath(readString(from)); + Path path = absPath(readString(conn.from)); logger->startWork(); auto & indirectRootStore = require(*store); indirectRootStore.addIndirectRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -704,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -718,24 +712,24 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : roots) size += i.second.size(); - to << size; + conn.to << size; for (auto & [target, links] : roots) for (auto & link : links) - to << link << store->printStorePath(target); + conn.to << link << store->printStorePath(target); break; } case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); + options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); - from >> options.ignoreLiveness >> options.maxFreed; + conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields - readInt(from); - readInt(from); - readInt(from); + readInt(conn.from); + readInt(conn.from); + readInt(conn.from); GCResults results; @@ -746,7 +740,7 @@ static void performOp(TunnelLogger * logger, ref store, gcStore.collectGarbage(options, results); logger->stopWork(); - to << results.paths << results.bytesFreed << 0 /* obsolete */; + conn.to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } @@ -755,24 +749,24 @@ static void performOp(TunnelLogger * logger, ref store, ClientSettings clientSettings; - clientSettings.keepFailed = readInt(from); - clientSettings.keepGoing = readInt(from); - clientSettings.tryFallback = readInt(from); - clientSettings.verbosity = (Verbosity) readInt(from); - clientSettings.maxBuildJobs = readInt(from); - clientSettings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - clientSettings.buildCores = readInt(from); - clientSettings.useSubstitutes = readInt(from); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); + clientSettings.keepFailed = readInt(conn.from); + clientSettings.keepGoing = readInt(conn.from); + clientSettings.tryFallback = readInt(conn.from); + clientSettings.verbosity = (Verbosity) readInt(conn.from); + clientSettings.maxBuildJobs = readInt(conn.from); + clientSettings.maxSilentTime = readInt(conn.from); + readInt(conn.from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(conn.from); + readInt(conn.from); // obsolete logType + readInt(conn.from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(conn.from); + clientSettings.useSubstitutes = readInt(conn.from); + + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { + unsigned int n = readInt(conn.from); for (unsigned int i = 0; i < n; i++) { - auto name = readString(from); - auto value = readString(from); + auto name = readString(conn.from); + auto value = readString(conn.from); clientSettings.overrides.emplace(name, value); } } @@ -789,20 +783,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) - to << 0; + conn.to << 0; else { - to << 1 + conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); WorkerProto::write(*store, wconn, i->second.references); - to << i->second.downloadSize - << i->second.narSize; + conn.to << i->second.downloadSize + << i->second.narSize; } break; } @@ -810,7 +804,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); @@ -819,12 +813,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); - to << infos.size(); + conn.to << infos.size(); for (auto & i : infos) { - to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + conn.to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); WorkerProto::write(*store, wconn, i.second.references); - to << i.second.downloadSize << i.second.narSize; + conn.to << i.second.downloadSize << i.second.narSize; } break; } @@ -838,22 +832,22 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); try { info = store->queryPathInfo(path); } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) throw; } logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) + conn.to << 1; WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); + conn.to << 0; } break; } @@ -862,61 +856,61 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->optimiseStore(); logger->stopWork(); - to << 1; + conn.to << 1; break; case WorkerProto::Op::VerifyStore: { bool checkContents, repair; - from >> checkContents >> repair; + conn.from >> checkContents >> repair; logger->startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, (RepairFlag) repair); logger->stopWork(); - to << errors; + conn.to << errors; break; } case WorkerProto::Op::AddSignatures: { - auto path = store->parseStorePath(readString(from)); - StringSet sigs = readStrings(from); + auto path = store->parseStorePath(readString(conn.from)); + StringSet sigs = readStrings(conn.from); logger->startWork(); store->addSignatures(path, sigs); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::NarFromPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + dumpPath(store->toRealPath(path), conn.to); break; } case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(from)); - auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); + auto path = store->parseStorePath(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - info.ca = ContentAddress::parseOpt(readString(from)); - from >> repair >> dontCheckSigs; + conn.from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } @@ -926,10 +920,10 @@ static void performOp(TunnelLogger * logger, ref store, else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + source = std::make_unique(conn.from, conn.to); else { - TeeSource tee { from, saved }; + TeeSource tee { conn.from, saved }; NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); @@ -957,15 +951,15 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::write(*store, wconn, willBuild); WorkerProto::write(*store, wconn, willSubstitute); WorkerProto::write(*store, wconn, unknown); - to << downloadSize << narSize; + conn.to << downloadSize << narSize; break; } case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + auto outputId = DrvOutput::parse(readString(conn.from)); + auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { @@ -978,10 +972,10 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); + auto outputId = DrvOutput::parse(readString(conn.from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); @@ -994,19 +988,19 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddBuildLog: { - StorePath path{readString(from)}; + StorePath path{readString(conn.from)}; logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); auto & logStore = require(*store); { - FramedSource source(from); + FramedSource source(conn.from); StringSink sink; source.drainInto(sink); logStore.addBuildLog(path, sink.s); } logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -1021,8 +1015,8 @@ static void performOp(TunnelLogger * logger, ref store, void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) { @@ -1038,7 +1032,12 @@ void processConnection( if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); - auto tunnelLogger = new TunnelLogger(to, clientVersion); + WorkerProto::BasicServerConnection conn; + conn.to = std::move(to); + conn.from = std::move(from); + conn.protoVersion = clientVersion; + + auto tunnelLogger = new TunnelLogger(conn.to, clientVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) @@ -1051,12 +1050,6 @@ void processConnection( printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); - WorkerProto::BasicServerConnection conn { - .to = to, - .from = from, - .clientVersion = clientVersion, - }; - conn.postHandshake(*store, { .daemonNixVersion = nixVersion, // We and the underlying store both need to trust the client for @@ -1072,13 +1065,13 @@ void processConnection( try { tunnelLogger->stopWork(); - to.flush(); + conn.to.flush(); /* Process client requests. */ while (true) { WorkerProto::Op op; try { - op = (enum WorkerProto::Op) readInt(from); + op = (enum WorkerProto::Op) readInt(conn.from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { @@ -1092,7 +1085,7 @@ void processConnection( debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the @@ -1108,19 +1101,19 @@ void processConnection( throw; } - to.flush(); + conn.to.flush(); assert(!tunnelLogger->state_.lock()->canSendStderr); }; } catch (Error & e) { tunnelLogger->stopWork(&e); - to.flush(); + conn.to.flush(); return; } catch (std::exception & e) { auto ex = Error(e.what()); tunnelLogger->stopWork(&ex); - to.flush(); + conn.to.flush(); return; } } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 1964c0d997c..a8ce32d8deb 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -10,8 +10,8 @@ enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5a518923bdc..3ebf4d6a966 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -10,6 +10,8 @@ #include #include +#include "strings-inline.hh" + namespace nix { std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 5cb245a7f0d..99ae5f48f0d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -8,13 +8,11 @@ #include "repair-flag.hh" #include "derived-path-map.hh" #include "sync.hh" -#include "comparator.hh" #include "variant-wrapper.hh" #include #include - namespace nix { struct StoreDirConfig; @@ -33,7 +31,8 @@ struct DerivationOutput { StorePath path; - GENERATE_CMP(InputAddressed, me->path); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -57,7 +56,8 @@ struct DerivationOutput */ StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - GENERATE_CMP(CAFixed, me->ca); + bool operator == (const CAFixed &) const = default; + auto operator <=> (const CAFixed &) const = default; }; /** @@ -77,7 +77,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashAlgo); + bool operator == (const CAFloating &) const = default; + auto operator <=> (const CAFloating &) const = default; }; /** @@ -85,7 +86,8 @@ struct DerivationOutput * isn't known yet. */ struct Deferred { - GENERATE_CMP(Deferred); + bool operator == (const Deferred &) const = default; + auto operator <=> (const Deferred &) const = default; }; /** @@ -104,7 +106,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashAlgo); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -117,7 +120,8 @@ struct DerivationOutput Raw raw; - GENERATE_CMP(DerivationOutput, me->raw); + bool operator == (const DerivationOutput &) const = default; + auto operator <=> (const DerivationOutput &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); @@ -178,7 +182,8 @@ struct DerivationType { */ bool deferred; - GENERATE_CMP(InputAddressed, me->deferred); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -202,7 +207,8 @@ struct DerivationType { */ bool fixed; - GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + bool operator == (const ContentAddressed &) const = default; + auto operator <=> (const ContentAddressed &) const = default; }; /** @@ -212,7 +218,8 @@ struct DerivationType { * type, but has some restrictions on its usage. */ struct Impure { - GENERATE_CMP(Impure); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -223,7 +230,8 @@ struct DerivationType { Raw raw; - GENERATE_CMP(DerivationType, me->raw); + bool operator == (const DerivationType &) const = default; + auto operator <=> (const DerivationType &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationType); @@ -319,14 +327,9 @@ struct BasicDerivation */ void applyRewrites(const StringMap & rewrites); - GENERATE_CMP(BasicDerivation, - me->outputs, - me->inputSrcs, - me->platform, - me->builder, - me->args, - me->env, - me->name); + bool operator == (const BasicDerivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BasicDerivation &) const = default; }; class Store; @@ -384,9 +387,9 @@ struct Derivation : BasicDerivation const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - GENERATE_CMP(Derivation, - static_cast(*me), - me->inputDrvs); + bool operator == (const Derivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const Derivation &) const = default; }; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 4c1ea417a36..c97d52773eb 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -54,17 +54,18 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode, - me->value, - me->childMap); +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept = default; -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>, - me->map); +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept = default; +#endif +template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 393cdedf747..bd60fe88710 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -47,7 +47,11 @@ struct DerivedPathMap { */ Map childMap; - DECLARE_CMP(ChildNode); + bool operator == (const ChildNode &) const noexcept; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // decltype(std::declval() <=> std::declval()) + // operator <=> (const ChildNode &) const noexcept; }; /** @@ -60,7 +64,10 @@ struct DerivedPathMap { */ Map map; - DECLARE_CMP(DerivedPathMap); + bool operator == (const DerivedPathMap &) const = default; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // auto operator <=> (const DerivedPathMap &) const noexcept; /** * Find the node for `k`, creating it if needed. @@ -83,14 +90,21 @@ struct DerivedPathMap { ChildNode * findSlot(const SingleDerivedPath & k); }; +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept; + +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept; + +template<> +inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +#endif -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::, - DerivedPathMap>); -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode::, - DerivedPathMap>::ChildNode); +extern template struct DerivedPathMap>::ChildNode; +extern template struct DerivedPathMap>; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index a7b40432108..1eef881de0c 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,6 +1,7 @@ #include "derived-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,26 +9,32 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) - -CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) - -CMP(SingleDerivedPath, DerivedPathBuilt, outputs) - -#undef CMP -#undef CMP_ONE +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleDerivedPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + DerivedPathBuilt ::, + DerivedPathBuilt, + *me->drvPath, + me->outputs); +GENERATE_ONE_CMP( + , + bool, + DerivedPathBuilt ::, + <, + DerivedPathBuilt, + *me->drvPath, + me->outputs); nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b238f844e56..4ba3fb37d4c 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,8 +3,8 @@ #include "path.hh" #include "outputs-spec.hh" -#include "comparator.hh" #include "config.hh" +#include "ref.hh" #include @@ -31,7 +31,8 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - GENERATE_CMP(DerivedPathOpaque, me->path); + bool operator == (const DerivedPathOpaque &) const = default; + auto operator <=> (const DerivedPathOpaque &) const = default; }; struct SingleDerivedPath; @@ -78,7 +79,8 @@ struct SingleDerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(SingleDerivedPathBuilt); + bool operator == (const SingleDerivedPathBuilt &) const noexcept; + std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept; }; using _SingleDerivedPathRaw = std::variant< @@ -108,6 +110,9 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { return static_cast(*this); } + bool operator == (const SingleDerivedPath &) const = default; + auto operator <=> (const SingleDerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). @@ -201,7 +206,9 @@ struct DerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(DerivedPathBuilt); + bool operator == (const DerivedPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const DerivedPathBuilt &) const noexcept; }; using _DerivedPathRaw = std::variant< @@ -230,6 +237,10 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + bool operator == (const DerivedPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const DerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 17ebaace6a8..c1e871e9384 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -6,6 +6,13 @@ namespace nix { struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + const std::string name() override { return "Dummy Store"; } std::string doc() override @@ -14,23 +21,24 @@ struct DummyStoreConfig : virtual StoreConfig { #include "dummy-store.md" ; } + + static std::set uriSchemes() { + return {"dummy"}; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : DummyStore(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - DummyStore(const Params & params) : StoreConfig(params) - , DummyStoreConfig(params) + , DummyStoreConfig(scheme, authority, params) , Store(params) { } + DummyStore(const Params & params) + : DummyStore("dummy", "", params) + { } + std::string getUri() override { return *uriSchemes().begin(); @@ -50,10 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return Trusted; } - static std::set uriSchemes() { - return {"dummy"}; - } - std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1c271cbec03..1f5b4ab934c 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,13 +1,15 @@ #pragma once ///@file -#include "types.hh" -#include "hash.hh" -#include "config.hh" - #include #include +#include "logging.hh" +#include "types.hh" +#include "ref.hh" +#include "config.hh" +#include "serialise.hh" + namespace nix { struct FileTransferSettings : Config diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index ab1059fb1ec..020f770b07a 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "store-api.hh" +#include +#include "store-api.hh" namespace nix { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e1d7ea6d67..fa4c0ba7fad 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -15,7 +15,6 @@ #include #ifndef _WIN32 -# include # include #endif @@ -35,6 +34,8 @@ #include #endif +#include "strings.hh" + namespace nix { @@ -82,7 +83,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); + builders = concatStringsSep("\n", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) @@ -333,60 +334,6 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } -Paths PluginFilesSetting::parse(const std::string & str) const -{ - if (pluginsLoaded) - throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - return BaseSetting::parse(str); -} - - -void initPlugins() -{ - assert(!settings.pluginFiles.pluginsLoaded); - for (const auto & pluginFile : settings.pluginFiles.get()) { - std::vector pluginFiles; - try { - auto ents = std::filesystem::directory_iterator{pluginFile}; - for (const auto & ent : ents) { - checkInterrupt(); - pluginFiles.emplace_back(ent.path()); - } - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - checkInterrupt(); - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ -#ifndef _WIN32 // TODO implement via DLL loading on Windows - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - - /* Older plugins use a statically initialized object to run their code. - Newer plugins can also export nix_plugin_entry() */ - void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); - if (nix_plugin_entry) - nix_plugin_entry(); -#else - throw Error("could not dynamically open plugin file '%s'", file); -#endif - } - } - - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); - - /* Tell the user if they try to set plugin-files after we've already loaded */ - settings.pluginFiles.pluginsLoaded = true; -} - static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dfe25f31726..30d7537bd52 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -31,23 +31,6 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int parse(const std::string & str) const override; }; -struct PluginFilesSetting : public BaseSetting -{ - bool pluginsLoaded = false; - - PluginFilesSetting(Config * options, - const Paths & def, - const std::string & name, - const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } - - Paths parse(const std::string & str) const override; -}; - const uint32_t maxIdsPerBuild = #if __linux__ 1 << 16 @@ -1158,33 +1141,6 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - PluginFilesSetting pluginFiles{ - this, {}, "plugin-files", - R"( - A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, - this symbol will be called. Alternatively, they can affect execution - through static initialization. In particular, these plugins may construct - static instances of RegisterPrimOp to add new primops or constants to the - expression language, RegisterStoreImplementation to add new store - implementations, RegisterCommand to add new subcommands to the `nix` - command, and RegisterSetting to add new nix config settings. See the - constructors for those types for more details. - - Warning! These APIs are inherently unstable and may change from - release to release. - - Since these files are loaded into the same address space as Nix - itself, they must be DSOs compatible with the instance of Nix - running at the time (i.e. compiled against the same headers, not - linked to any incompatible libraries). They should not be linked to - any Nix libs directly, as those will be available already at load - time. - - If an entry in the list is a directory, all files in the directory - are loaded as plugins (non-recursively). - )"}; - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -1278,12 +1234,6 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/** - * This should be called after settings are initialized, but before - * anything else - */ -void initPlugins(); - /** * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the * given configuration object. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3328caef9c4..b15ef4e4cba 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "http-binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -8,26 +8,37 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + std::string_view scheme, + std::string_view _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , cacheUri( + std::string { scheme } + + "://" + + (!_cacheUri.empty() + ? _cacheUri + : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + while (!cacheUri.empty() && cacheUri.back() == '/') + cacheUri.pop_back(); +} - const std::string name() override { return "HTTP Binary Cache Store"; } - std::string doc() override - { - return - #include "http-binary-cache-store.md" - ; - } -}; +std::string HttpBinaryCacheStoreConfig::doc() +{ + return + #include "http-binary-cache-store.md" + ; +} + class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: - Path cacheUri; - struct State { bool enabled = true; @@ -40,23 +51,14 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v HttpBinaryCacheStore( std::string_view scheme, - PathView _cacheUri, + PathView cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) , Store(params) , BinaryCacheStore(params) - , cacheUri( - std::string { scheme } - + "://" - + (!_cacheUri.empty() - ? _cacheUri - : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - while (!cacheUri.empty() && cacheUri.back() == '/') - cacheUri.pop_back(); - diskCache = getNarInfoDiskCache(); } @@ -81,14 +83,6 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - protected: void maybeDisable() diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 00000000000..d2fc43210a2 --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,30 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + + Path cacheUri; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 9664b126edc..eac360a4f7a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,5 +1,5 @@ #include "legacy-ssh-store.hh" -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -15,6 +15,15 @@ namespace nix { +LegacySSHStoreConfig::LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) +{ +} + std::string LegacySSHStoreConfig::doc() { return @@ -35,7 +44,7 @@ LegacySSHStore::LegacySSHStore( const Params & params) : StoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(params) + , LegacySSHStoreConfig(scheme, host, params) , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index db49188ec9d..b541455b4e5 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "store-api.hh" #include "ssh.hh" #include "callback.hh" @@ -13,6 +13,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; + LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; @@ -21,6 +26,8 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const std::string name() override { return "SSH Store"; } + static std::set uriSchemes() { return {"ssh"}; } + std::string doc() override; }; @@ -41,8 +48,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - static std::set uriSchemes() { return {"ssh"}; } - LegacySSHStore( std::string_view scheme, std::string_view host, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index aa2efdb8fb6..dcc6affe4a1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" @@ -7,28 +7,27 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( + std::string_view scheme, + PathView binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , binaryCacheDir(binaryCacheDir) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - const std::string name() override { return "Local Binary Cache Store"; } +} - std::string doc() override - { - return - #include "local-binary-cache-store.md" - ; - } -}; -class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore +std::string LocalBinaryCacheStoreConfig::doc() { -private: - - Path binaryCacheDir; + return + #include "local-binary-cache-store.md" + ; +} -public: +struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +{ /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. @@ -39,10 +38,9 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) , Store(params) , BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) { } @@ -53,8 +51,6 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public return "file://" + binaryCacheDir; } - static std::set uriSchemes(); - protected: bool fileExists(const std::string & path) override; @@ -123,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStore::uriSchemes() +std::set LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 00000000000..997e8ecbb51 --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,23 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + + Path binaryCacheDir; + + const std::string name() override + { + return "Local Binary Cache Store"; + } + + static std::set uriSchemes(); + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 843c0d28881..5449b20eb3b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -8,6 +8,20 @@ namespace nix { +LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) + : StoreConfig(params) + // Default `?root` from `rootDir` if non set + // FIXME don't duplicate description once we don't have root setting + , rootDir{ + this, + !rootDir.empty() && params.count("root") == 0 + ? (std::optional{rootDir}) + : std::nullopt, + "root", + "Directory prefixed to all other paths."} +{ +} + LocalFSStore::LocalFSStore(const Params & params) : Store(params) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8fb08120012..9bb569f0b25 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,6 +11,15 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const Params & params); + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 598415db837..ec2c5f4e936 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -18,11 +18,11 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Params & params) +LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(path, params) , LocalStoreConfig(params) - , LocalOverlayStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , LocalStore(params) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 35a30101391..63628abed50 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -8,11 +8,16 @@ namespace nix { struct LocalOverlayStoreConfig : virtual LocalStoreConfig { LocalOverlayStoreConfig(const StringMap & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + : LocalOverlayStoreConfig("local-overlay", "", params) { } + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) + { + } + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) @@ -58,6 +63,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } + static std::set uriSchemes() + { + return { "local-overlay" }; + } + std::string doc() override; protected: @@ -90,19 +100,12 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual ref lowerStore; public: - LocalOverlayStore(const Params & params); - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : LocalOverlayStore(params) + LocalOverlayStore(const Params & params) + : LocalOverlayStore("local-overlay", "", params) { - if (!path.empty()) - throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } - static std::set uriSchemes() - { - return { "local-overlay" }; - } + LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); std::string getUri() override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2b4e01eb3ba..819cee34532 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -51,9 +51,20 @@ #include +#include "strings.hh" + namespace nix { +LocalStoreConfig::LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(authority, params) +{ +} + std::string LocalStoreConfig::doc() { return @@ -181,10 +192,13 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Params & params) +LocalStore::LocalStore( + std::string_view scheme, + PathView path, + const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , dbDir(stateDir + "/db") @@ -463,19 +477,8 @@ LocalStore::LocalStore(const Params & params) } -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & _params) - : LocalStore([&]{ - // Default `?root` from `path` if non set - if (!path.empty() && _params.count("root") == 0) { - auto params = _params; - params.insert_or_assign("root", std::string { path }); - return params; - } - return _params; - }()) +LocalStore::LocalStore(const Params & params) + : LocalStore("local", "", params) { } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index b0a0def9aae..a03cfc03b30 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,6 +38,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + Setting requireSigs{this, settings.requireSigs, "require-sigs", @@ -62,6 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig const std::string name() override { return "Local Store"; } + static std::set uriSchemes() + { return {"local"}; } + std::string doc() override; }; @@ -144,9 +152,6 @@ public: ~LocalStore(); - static std::set uriSchemes() - { return {"local"}; } - /** * Implementations of abstract store API methods. */ diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 2a55c445667..983652d5f8b 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" +#include "ref.hh" #include "store-reference.hh" namespace nix { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 7444cba20b5..29ee95b75bc 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -99,18 +99,14 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite +subdir('build-utils-meson/generate-header') + generated_headers = [] foreach header : [ 'schema.sql', 'ca-specific-schema.sql', ] - generated_headers += custom_target( - command : [ 'bash', '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], - input : header, - output : '@PLAINNAME@.gen.hh', - install : true, - install_dir : get_option('includedir') / 'nix', - ) + generated_headers += gen_header.process(header) endforeach busybox = find_program(get_option('sandbox-shell'), required : false) @@ -166,6 +162,7 @@ sources = files( 'builtins/fetchurl.cc', 'builtins/unpack-channel.cc', 'common-protocol.cc', + 'common-ssh-store-config.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', @@ -210,7 +207,6 @@ sources = files( 'serve-protocol-connection.cc', 'serve-protocol.cc', 'sqlite.cc', - 'ssh-store-config.cc', 'ssh-store.cc', 'ssh.cc', 'store-api.cc', @@ -237,6 +233,7 @@ headers = [config_h] + files( 'builtins/buildenv.hh', 'common-protocol-impl.hh', 'common-protocol.hh', + 'common-ssh-store-config.hh', 'content-address.hh', 'daemon.hh', 'derivations.hh', @@ -246,10 +243,12 @@ headers = [config_h] + files( 'filetransfer.hh', 'gc-store.hh', 'globals.hh', + 'http-binary-cache-store.hh', 'indirect-root-store.hh', 'keys.hh', 'legacy-ssh-store.hh', 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', 'local-fs-store.hh', 'local-overlay-store.hh', 'local-store.hh', @@ -276,11 +275,11 @@ headers = [config_h] + files( 'remote-store.hh', 's3-binary-cache-store.hh', 's3.hh', + 'ssh-store.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', 'serve-protocol.hh', 'sqlite.hh', - 'ssh-store-config.hh', 'ssh.hh', 'store-api.hh', 'store-cast.hh', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cc3f4884f51..bcc02206bc9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,3 +1,5 @@ +#include + #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" @@ -8,6 +10,7 @@ #include "callback.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b1079b027d6..9a541bb7776 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -3,7 +3,6 @@ #include #include -#include #include @@ -74,7 +73,10 @@ struct NarAccessor : public SourceAccessor NarMember & createMember(const CanonPath & path, NarMember member) { size_t level = 0; - for (auto _ : path) ++level; + for (auto _ : path) { + (void)_; + ++level; + } while (parents.size() > level) parents.pop(); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 07beb8acb0b..288f618d5d7 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -7,6 +7,8 @@ #include #include +#include "strings.hh" + namespace nix { static const char * schema = R"sql( diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 0d219a48929..2442a7b09a3 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,18 +1,10 @@ #include "globals.hh" #include "nar-info.hh" #include "store-api.hh" +#include "strings.hh" namespace nix { -GENERATE_CMP_EXT( - , - NarInfo, - me->url, - me->compression, - me->fileHash, - me->fileSize, - static_cast(*me)); - NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index fd538a7cd9f..561c9a86364 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,7 +24,9 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - DECLARE_CMP(NarInfo); + bool operator ==(const NarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + //auto operator <=>(const NarInfo &) const = default; std::string to_string(const Store & store) const; diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c06922379..86788a87e72 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,6 +5,7 @@ #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 1ef99a5fc67..30d15311d0a 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,9 +6,7 @@ #include #include -#include "comparator.hh" #include "json-impls.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -60,7 +58,11 @@ struct OutputsSpec { Raw raw; - GENERATE_CMP(OutputsSpec, me->raw); + bool operator == (const OutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const OutputsSpec & other) const { + return raw < other.raw; + } MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); @@ -99,7 +101,9 @@ struct ExtendedOutputsSpec { Raw raw; - GENERATE_CMP(ExtendedOutputsSpec, me->raw); + bool operator == (const ExtendedOutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const ExtendedOutputsSpec &) const; MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5c27182b720..6e87e60f446 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,11 +3,14 @@ #include "path-info.hh" #include "store-api.hh" #include "json-utils.hh" +#include "comparator.hh" +#include "strings.hh" namespace nix { GENERATE_CMP_EXT( , + std::weak_ordering, UnkeyedValidPathInfo, me->deriver, me->narHash, @@ -19,12 +22,6 @@ GENERATE_CMP_EXT( me->sigs, me->ca); -GENERATE_CMP_EXT( - , - ValidPathInfo, - me->path, - static_cast(*me)); - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b6dc0855d75..71f1476a672 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,17 +32,47 @@ struct SubstitutablePathInfo using SubstitutablePathInfos = std::map; +/** + * Information about a store object. + * + * See `store/store-object` and `protocols/json/store-object-info` in + * the Nix manual + */ struct UnkeyedValidPathInfo { + /** + * Path to derivation that produced this store object, if known. + */ std::optional deriver; + /** * \todo document this */ Hash narHash; + + /** + * Other store objects this store object referes to. + */ StorePathSet references; + + /** + * When this store object was registered in the store that contains + * it, if known. + */ time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id = 0; // internal use only + + /** + * 0 = unknown + */ + uint64_t narSize = 0; + + /** + * internal use only: SQL primary key for on-disk store objects with + * `LocalStore`. + * + * @todo Remove, layer violation + */ + uint64_t id = 0; /** * Whether the path is ultimately trusted, that is, it's a @@ -75,7 +105,12 @@ struct UnkeyedValidPathInfo UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; - DECLARE_CMP(UnkeyedValidPathInfo); + bool operator == (const UnkeyedValidPathInfo &) const noexcept; + + /** + * @todo return `std::strong_ordering` once `id` is removed + */ + std::weak_ordering operator <=> (const UnkeyedValidPathInfo &) const noexcept; virtual ~UnkeyedValidPathInfo() { } @@ -95,7 +130,8 @@ struct UnkeyedValidPathInfo struct ValidPathInfo : UnkeyedValidPathInfo { StorePath path; - DECLARE_CMP(ValidPathInfo); + bool operator == (const ValidPathInfo &) const = default; + auto operator <=> (const ValidPathInfo &) const = default; /** * Return a fingerprint of the store path to be used in binary @@ -135,6 +171,9 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + /** + * References as store path basenames, including a self reference if it has one. + */ Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 026e376471f..161d023d14c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,7 +1,9 @@ +#include + #include "path-with-outputs.hh" #include "store-api.hh" +#include "strings.hh" -#include namespace nix { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d749ccd0a35..ebb0864c555 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,7 +73,7 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.daemonVersion = WorkerProto::BasicClientConnection::handshake( + conn.protoVersion = WorkerProto::BasicClientConnection::handshake( conn.to, tee, PROTOCOL_VERSION); } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close @@ -115,7 +115,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); @@ -128,7 +128,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); - overrides.erase(settings.pluginFiles.name); + overrides.erase("plugin-files"); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -175,7 +175,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); @@ -198,7 +198,7 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) { conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); @@ -221,7 +221,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { for (auto & i : pathsMap) { SubstitutablePathInfo info; @@ -241,7 +241,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); @@ -368,7 +368,7 @@ ref RemoteStore::addCAToStore( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { conn->to << WorkerProto::Op::AddToStore @@ -485,7 +485,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows ; @@ -513,11 +513,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -554,7 +554,7 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { - if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { auto conn(getConnection()); conn->to << WorkerProto::Op::AddMultipleToStore @@ -572,7 +572,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { @@ -587,7 +587,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -597,7 +597,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto outPaths = WorkerProto::Serialise>::read( *this, *conn); if (outPaths.empty()) @@ -644,9 +644,9 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); WorkerProto::write(*this, *conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we @@ -667,7 +667,7 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; WorkerProto::write(*this, *conn, paths); conn->to << buildMode; @@ -841,7 +841,7 @@ void RemoteStore::queryMissing(const std::vector & targets, { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; @@ -889,7 +889,7 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->daemonVersion; + return conn->protoVersion; } std::optional RemoteStore::isTrustedClient() diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27013c0f17a..1a0ec11112e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,7 @@ #if ENABLE_S3 +#include + #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -190,76 +192,31 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) , BinaryCacheStore(params) { } -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , bucketName(bucketName) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - const Setting profile{this, "", "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - - const Setting region{this, Aws::Region::US_EAST_1, "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{this, "", "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{this, "", "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{this, "", "narinfo-compression", - "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", - "Compression method for `.ls` files."}; - - const Setting logCompression{this, "", "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{ - this, false, "multipart-upload", - "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", - "Size (in bytes) of each part in multi-part uploads."}; - - const std::string name() override { return "S3 Binary Cache Store"; } - - std::string doc() override - { - return - #include "s3-binary-cache-store.md" - ; - } -}; + // Don't want to use use AWS SDK in header, so we check the default + // here. TODO do this better after we overhaul the store settings + // system. + assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + + if (bucketName.empty()) + throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); +} + +std::string S3BinaryCacheStoreConfig::doc() +{ + return + #include "s3-binary-cache-store.md" + ; +} + struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { @@ -275,15 +232,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) , Store(params) , BinaryCacheStore(params) , S3BinaryCacheStore(params) - , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { - if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); diskCache = getNarInfoDiskCache(); } @@ -521,9 +475,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual { return std::nullopt; } - - static std::set uriSchemes() { return {"s3"}; } - }; static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c62ea5147f3..7d303a115f4 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,6 +7,101 @@ namespace nix { +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + std::string bucketName; + + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + +protected: + + constexpr static const char * defaultRegion = "us-east-1"; + +public: + + const Setting region{ + this, + defaultRegion, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + const std::string name() override + { + return "S3 Binary Cache Store"; + } + + static std::set uriSchemes() + { + return {"s3"}; + } + + std::string doc() override; +}; + class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index 67bc5dc6eed..6f3b177acd2 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -10,7 +10,6 @@ #include "serve-protocol.hh" #include "length-prefixed-protocol-helper.hh" -#include "store-api.hh" namespace nix { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 7ad934b738b..954a9746774 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,7 +1,5 @@ -#include "ssh-store-config.hh" -#include "store-api.hh" +#include "ssh-store.hh" #include "local-fs-store.hh" -#include "remote-store.hh" #include "remote-store-connection.hh" #include "source-accessor.hh" #include "archive.hh" @@ -12,23 +10,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using RemoteStoreConfig::RemoteStoreConfig; - using CommonSSHStoreConfig::CommonSSHStoreConfig; - - const Setting remoteProgram{this, {"nix-daemon"}, "remote-program", - "Path to the `nix-daemon` executable on the remote machine."}; - - const std::string name() override { return "Experimental SSH Store"; } +} - std::string doc() override - { - return - #include "ssh-store.md" - ; - } -}; +std::string SSHStoreConfig::doc() +{ + return + #include "ssh-store.md" + ; +} class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { @@ -41,7 +38,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) + , SSHStoreConfig(scheme, host, params) , Store(params) , RemoteStore(params) , master(createSSHMaster( @@ -50,8 +47,6 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { } - static std::set uriSchemes() { return {"ssh-ng"}; } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; @@ -92,43 +87,32 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig -{ - using SSHStoreConfig::SSHStoreConfig; - using LocalFSStoreConfig::LocalFSStoreConfig; - - MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } +MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - const std::string name() override { return "Experimental SSH Store with filesystem mounted"; } +MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - std::string doc() override - { - return - #include "mounted-ssh-store.md" - ; - } +std::string MountedSSHStoreConfig::doc() +{ + return + #include "mounted-ssh-store.md" + ; +} - std::optional experimentalFeature() const override - { - return ExperimentalFeature::MountedSSHStore; - } -}; /** * The mounted ssh store assumes that filesystems on the remote host are @@ -168,11 +152,6 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH }; } - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh new file mode 100644 index 00000000000..29a2a8b2c2d --- /dev/null +++ b/src/libstore/ssh-store.hh @@ -0,0 +1,61 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "local-fs-store.hh" +#include "remote-store.hh" + +namespace nix { + +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + + const Setting remoteProgram{ + this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + + const std::string name() override + { + return "Experimental SSH Store"; + } + + static std::set uriSchemes() + { + return {"ssh-ng"}; + } + + std::string doc() override; +}; + +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + using SSHStoreConfig::SSHStoreConfig; + + MountedSSHStoreConfig(StringMap params); + + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + + const std::string name() override + { + return "Experimental SSH Store with filesystem mounted"; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string doc() override; + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 05c4e1c5e8f..2c4dee518b4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -22,6 +22,8 @@ #include #include +#include "strings.hh" + using json = nlohmann::json; namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a5effb4c131..7d5f533c5a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -18,14 +18,10 @@ #include #include -#include #include -#include -#include #include #include #include -#include namespace nix { @@ -220,6 +216,10 @@ public: virtual ~Store() { } + /** + * @todo move to `StoreConfig` one we store enough information in + * those to recover the scheme and authority in all cases. + */ virtual std::string getUri() = 0; /** @@ -901,7 +901,7 @@ struct Implementations { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .create = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index e99335c0d57..459cea9c20e 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -71,7 +71,6 @@ struct StoreReference Params params; bool operator==(const StoreReference & rhs) const = default; - auto operator<=>(const StoreReference & rhs) const = default; /** * Render the whole store reference as a URI, including parameters. diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 499f7696712..3c445eb1318 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -2,10 +2,8 @@ #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include #include #include -#include #include #include @@ -19,6 +17,21 @@ namespace nix { +UDSRemoteStoreConfig::UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , path{authority.empty() ? settings.nixDaemonSocketFile : authority} +{ + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } +} + + std::string UDSRemoteStoreConfig::doc() { return @@ -27,11 +40,20 @@ std::string UDSRemoteStoreConfig::doc() } +// A bit gross that we now pass empty string but this is knowing that +// empty string will later default to the same nixDaemonSocketFile. Why +// don't we just wire it all through? I believe there are cases where it +// will live reload so we want to continue to account for that. UDSRemoteStore::UDSRemoteStore(const Params & params) + : UDSRemoteStore(scheme, "", params) +{} + + +UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) + , UDSRemoteStoreConfig(scheme, authority, params) , Store(params) , LocalFSStore(params) , RemoteStore(params) @@ -39,25 +61,15 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } -UDSRemoteStore::UDSRemoteStore( - std::string_view scheme, - PathView socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - if (!socket_path.empty()) - path.emplace(socket_path); -} - - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string("unix://") + *path; - } else { - // unix:// with no path also works. Change what we return? - return "daemon"; - } + return path == settings.nixDaemonSocketFile + ? // FIXME: Not clear why we return daemon here and not default + // to settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + "daemon" + : std::string(scheme) + "://" + path; } @@ -74,7 +86,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path ? *path : settings.nixDaemonSocketFile); + nix::connect(toSocket(conn->fd.get()), path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 6f0494bb63b..a8e57166416 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,16 +9,37 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern + // outlined in https://github.com/NixOS/nix/issues/10766 + using LocalFSStoreConfig::LocalFSStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + /** + * @param authority is the socket path. + */ + UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); const std::string name() override { return "Local Daemon Store"; } std::string doc() override; + + /** + * The path to the unix domain socket. + * + * The default is `settings.nixDaemonSocketFile`, but we don't write + * that below, instead putting in the constructor. + */ + Path path; + +protected: + static constexpr char const * scheme = "unix"; + +public: + static std::set uriSchemes() + { return {scheme}; } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig @@ -27,17 +48,21 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig { public: + /** + * @deprecated This is the old API to construct the store. + */ UDSRemoteStore(const Params & params); + + /** + * @param authority is the socket path. + */ UDSRemoteStore( std::string_view scheme, - PathView path, + std::string_view authority, const Params & params); std::string getUri() override; - static std::set uriSchemes() - { return {"unix"}; } - ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } @@ -63,7 +88,6 @@ private: }; ref openConnection() override; - std::optional path; }; } diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index dfc20879881..d73d86ff27f 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -3,6 +3,7 @@ #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index d5a3e0034f9..0dd102200a5 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -64,6 +64,8 @@ #include #include +#include "strings.hh" + namespace nix { void handleDiffHook( @@ -175,7 +177,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() +Goal::Co LocalDerivationGoal::tryLocalBuild() { #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); @@ -183,10 +185,10 @@ void LocalDerivationGoal::tryLocalBuild() unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { - state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); } assert(derivationType); @@ -240,7 +242,8 @@ void LocalDerivationGoal::tryLocalBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryLocalBuild(); } } @@ -255,15 +258,13 @@ void LocalDerivationGoal::tryLocalBuild() outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, std::move(e)); - return; + co_return done(BuildResult::InputRejected, {}, std::move(e)); } - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); + co_await Suspend{}; + // after EOF on child + co_return buildDone(); } static void chmod_(const Path & path, mode_t mode) @@ -1525,10 +1526,11 @@ void LocalDerivationGoal::startDaemon() debug("received daemon connection"); auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); try { - daemon::processConnection(store, from, to, + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SystemError &) { diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index 4bcf5c9d457..bf25cf2a60b 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -198,7 +198,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - void tryLocalBuild() override; + Goal::Co tryLocalBuild() override; /** * Start building a derivation. diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index 8057aa13e1b..29f4b2cb31c 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,12 +1,13 @@ +#include +#include +#include + #include "user-lock.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" #include "users.hh" -#include -#include - namespace nix { #if __linux__ diff --git a/src/libstore/unix/user-lock.hh b/src/libstore/unix/user-lock.hh index 1c268e1fbd5..a7caf8518f3 100644 --- a/src/libstore/unix/user-lock.hh +++ b/src/libstore/unix/user-lock.hh @@ -1,10 +1,8 @@ #pragma once ///@file -#include "types.hh" - -#include - +#include +#include #include namespace nix { diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 072bae8da82..93d13d48e55 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -58,7 +58,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink } else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { ex = std::make_exception_ptr(readError(from)); } else { auto error = readString(from); @@ -114,7 +114,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink // explain to users what's going on when their daemon is // older than #4628 (2023). if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) - && GET_PROTOCOL_MINOR(daemonVersion) <= 35) { + && GET_PROTOCOL_MINOR(protoVersion) <= 35) { auto m = e.msg(); if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) @@ -152,7 +152,6 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, throw Error("Nix daemon protocol version not supported"); if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - to << localVersion; return std::min(daemonVersion, localVersion); } @@ -173,15 +172,15 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha { WorkerProto::ClientHandshakeInfo res; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { // Obsolete CPU affinity. to << 0; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) to << false; // obsolete reserveSpace - if (GET_PROTOCOL_MINOR(daemonVersion) >= 33) + if (GET_PROTOCOL_MINOR(protoVersion) >= 33) to.flush(); return WorkerProto::Serialise::read(store, *this); @@ -189,12 +188,12 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { // Obsolete CPU affinity. readInt(from); } - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) readInt(from); // obsolete reserveSpace WorkerProto::write(store, *this, info); @@ -212,7 +211,7 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( throw InvalidPath(std::move(e.info())); throw; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 17) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { bool valid; from >> valid; if (!valid) @@ -224,10 +223,10 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - assert(GET_PROTOCOL_MINOR(daemonVersion) >= 12); + assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(store, *this, paths); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 27) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { to << maybeSubstitute; } processStderr(daemonException); diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 9dd723fd089..38287d08e31 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -6,7 +6,7 @@ namespace nix { -struct WorkerProto::BasicClientConnection +struct WorkerProto::BasicConnection { /** * Send with this. @@ -19,14 +19,45 @@ struct WorkerProto::BasicClientConnection FdSource from; /** - * Worker protocol version used for the connection. + * The protocol version agreed by both sides. + */ + WorkerProto::Version protoVersion; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn() + { + return WorkerProto::ReadConn{ + .from = from, + .version = protoVersion, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) + * The serve protocol connection types are unidirectional, unlike + * this type. */ - WorkerProto::Version daemonVersion; + operator WorkerProto::WriteConn() + { + return WorkerProto::WriteConn{ + .to = to, + .version = protoVersion, + }; + } +}; +struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection +{ /** * Flush to direction */ @@ -60,38 +91,6 @@ struct WorkerProto::BasicClientConnection */ ClientHandshakeInfo postHandshake(const StoreDirConfig & store); - /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::ReadConn() - { - return WorkerProto::ReadConn{ - .from = from, - .version = daemonVersion, - }; - } - - /** - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::WriteConn() - { - return WorkerProto::WriteConn{ - .to = to, - .version = daemonVersion, - }; - } - void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); StorePathSet queryValidPaths( @@ -124,43 +123,8 @@ struct WorkerProto::BasicClientConnection void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); }; -struct WorkerProto::BasicServerConnection +struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection { - /** - * Send with this. - */ - FdSink & to; - - /** - * Receive with this. - */ - FdSource & from; - - /** - * Worker protocol version used for the connection. - * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) - */ - WorkerProto::Version clientVersion; - - operator WorkerProto::ReadConn() - { - return WorkerProto::ReadConn{ - .from = from, - .version = clientVersion, - }; - } - - operator WorkerProto::WriteConn() - { - return WorkerProto::WriteConn{ - .to = to, - .version = clientVersion, - }; - } - /** * Establishes connection, negotiating version. * diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 62a12d18201..9fc16d0153e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -82,6 +82,7 @@ struct WorkerProto * * @todo remove once Hydra uses Store abstraction consistently. */ + struct BasicConnection; struct BasicClientConnection; struct BasicServerConnection; @@ -183,8 +184,7 @@ enum struct WorkerProto::Op : uint64_t struct WorkerProto::ClientHandshakeInfo { /** - * The version of the Nix daemon that is processing our requests -. + * The version of the Nix daemon that is processing our requests. * * Do note, it may or may not communicating with another daemon, * rather than being an "end" `LocalStore` or similar. diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c202facdfea..d58f4b4aeaf 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -57,8 +57,7 @@ void Completions::add(std::string completion, std::string description) }); } -bool Completion::operator<(const Completion & other) const -{ return completion < other.completion || (completion == other.completion && description < other.description); } +auto Completion::operator<=>(const Completion & other) const noexcept = default; std::string completionMarker = "___COMPLETE___"; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7759b74a93c..c0236ee3d72 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include #include #include #include @@ -11,6 +10,7 @@ #include "types.hh" #include "experimental-features.hh" +#include "ref.hh" namespace nix { @@ -380,7 +380,7 @@ struct Completion { std::string completion; std::string description; - bool operator<(const Completion & other) const; + auto operator<=>(const Completion & other) const noexcept; }; /** diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index 5c55c37a55d..34a43b53835 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,6 +29,7 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { +protected: /** * @brief The command's "working directory", but only set when top level. * diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 27f048697e5..03db6378a82 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,6 +1,7 @@ #include "canon-path.hh" #include "util.hh" #include "file-path-impl.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 8f5a1c2793d..f84347dc458 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -169,7 +169,7 @@ public: * a directory is always followed directly by its children. For * instance, 'foo' < 'foo/bar' < 'foo!'. */ - bool operator < (const CanonPath & x) const + auto operator <=> (const CanonPath & x) const { auto i = path.begin(); auto j = x.path.begin(); @@ -178,10 +178,9 @@ public: if (c_i == '/') c_i = 0; auto c_j = *j; if (c_j == '/') c_j = 0; - if (c_i < c_j) return true; - if (c_i > c_j) return false; + if (auto cmp = c_i <=> c_j; cmp != 0) return cmp; } - return i == path.end() && j != x.path.end(); + return (i != path.end()) <=> (j != x.path.end()); } /** diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd72..34ba6f4534c 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,17 +1,8 @@ #pragma once ///@file -#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, ==, my_type) -#define DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, <, my_type) -#define DECLARE_NEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, !=, my_type) - -#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ +#define GENERATE_ONE_CMP(PRE, RET, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE RET QUAL operator COMPARATOR(const MY_TYPE & other) const noexcept { \ __VA_OPT__(const MY_TYPE * me = this;) \ auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ @@ -19,30 +10,9 @@ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) -#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) -#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) - -/** - * Declare comparison methods without defining them. - */ -#define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(,,my_type) \ - DECLARE_LEQ(,,my_type) \ - DECLARE_NEQ(,,my_type) - -/** - * @param prefix This is for something before each declaration like - * `template`. - * - * @param my_type the type are defining operators for. - */ -#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ - DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_NEQ(prefix, qualification, my_type) + GENERATE_ONE_CMP(prefix, bool, qualification, ==, my_type, args) +#define GENERATE_SPACESHIP(prefix, ret, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, ret, qualification, <=>, my_type, args) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -55,15 +25,19 @@ * will generate comparison operators semantically equivalent to: * * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; + * auto operator<=>(const ClassName& other) const noexcept { + * if (auto cmp = field1 <=> other.field1; cmp != 0) + * return cmp; + * if (auto cmp = field2 <=> other.field2; cmp != 0) + * return cmp; + * ... + * return 0; * } * ``` */ #define GENERATE_CMP(args...) \ GENERATE_EQUAL(,,args) \ - GENERATE_LEQ(,,args) \ - GENERATE_NEQ(,,args) + GENERATE_SPACESHIP(,auto,,args) /** * @param prefix This is for something before each declaration like @@ -71,7 +45,6 @@ * * @param my_type the type are defining operators for. */ -#define GENERATE_CMP_EXT(prefix, my_type, args...) \ +#define GENERATE_CMP_EXT(prefix, ret, my_type, args...) \ GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ - GENERATE_LEQ(prefix, my_type ::, my_type, args) \ - GENERATE_NEQ(prefix, my_type ::, my_type, args) + GENERATE_SPACESHIP(prefix, ret, my_type ::, my_type, args) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 907ca7fc149..726e5091e54 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + namespace nix { Config::Config(StringMap initials) @@ -114,7 +116,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto include = false; auto ignoreMissing = false; @@ -127,7 +129,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p if (include) { if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { try { @@ -143,7 +145,7 @@ static void parseConfigFiles(const std::string & contents, const std::string & p } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); std::string name = std::move(tokens[0]); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e01f064484c..33c391963f3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -46,27 +46,22 @@ std::ostream & operator <<(std::ostream & os, const HintFmt & hf) /** * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. */ -inline bool operator<(const Trace& lhs, const Trace& rhs) +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs) { // `std::shared_ptr` does not have value semantics for its comparison // functions, so we need to check for nulls and compare the dereferenced // values here. if (lhs.pos != rhs.pos) { - if (!lhs.pos) - return true; - if (!rhs.pos) - return false; - if (*lhs.pos != *rhs.pos) - return *lhs.pos < *rhs.pos; + if (auto cmp = bool{lhs.pos} <=> bool{rhs.pos}; cmp != 0) + return cmp; + if (auto cmp = *lhs.pos <=> *rhs.pos; cmp != 0) + return cmp; } // This formats a freshly formatted hint string and then throws it away, which // shouldn't be much of a problem because it only runs when pos is equal, and this function is // used for trace printing, which is infrequent. - return lhs.hint.str() < rhs.hint.str(); + return lhs.hint.str() <=> rhs.hint.str(); } -inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } -inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } -inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 4b08a045e32..d7fe902d6c0 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,16 +16,12 @@ */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include -#include #include #include @@ -79,10 +75,7 @@ struct Trace { TracePrint print = TracePrint::Default; }; -inline bool operator<(const Trace& lhs, const Trace& rhs); -inline bool operator> (const Trace& lhs, const Trace& rhs); -inline bool operator<=(const Trace& lhs, const Trace& rhs); -inline bool operator>=(const Trace& lhs, const Trace& rhs); +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; @@ -127,6 +120,8 @@ protected: public: BaseError(const BaseError &) = default; + BaseError& operator=(const BaseError &) = default; + BaseError& operator=(BaseError &&) = default; template BaseError(unsigned int status, const Args & ... args) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 1da2a3ff55d..6ffbc0c1028 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "error.hh" #include "json-utils.hh" #include "types.hh" diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index 4c7218f1941..ec42d3d3493 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -2,8 +2,6 @@ ///@file #include "source-accessor.hh" -#include "fs-sink.hh" -#include "util.hh" namespace nix { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f75851bbd58..9042e3a5e67 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -23,6 +23,8 @@ # include #endif +#include "strings-inline.hh" + namespace fs = std::filesystem; namespace nix { diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index cf7d34d2273..de165fb7d5f 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "serialise.hh" #include "source-accessor.hh" #include "file-system.hh" @@ -41,6 +40,19 @@ struct FileSystemObjectSink virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; }; +/** + * An extension of `FileSystemObjectSink` that supports file types + * that are not supported by Nix's FSO model. + */ +struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink +{ + /** + * Create a hard link. The target must be the path of a previously + * encountered file relative to the root of the FSO. + */ + virtual void createHardlink(const CanonPath & path, const CanonPath & target) = 0; +}; + /** * Recursively copy file system objects from the source into the sink. */ diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 2190cc5509c..1dbdb733582 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -39,7 +39,8 @@ struct TreeEntry Mode mode; Hash hash; - GENERATE_CMP(TreeEntry, me->mode, me->hash); + bool operator ==(const TreeEntry &) const = default; + auto operator <=>(const TreeEntry &) const = default; }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7064e96e61c..35b913e4249 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -41,7 +41,7 @@ Hash::Hash(HashAlgorithm algo) : algo(algo) } -bool Hash::operator == (const Hash & h2) const +bool Hash::operator == (const Hash & h2) const noexcept { if (hashSize != h2.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) @@ -50,7 +50,7 @@ bool Hash::operator == (const Hash & h2) const } -std::strong_ordering Hash::operator <=> (const Hash & h) const +std::strong_ordering Hash::operator <=> (const Hash & h) const noexcept { if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ef96d08c978..dc95b9f2f9b 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -88,12 +88,12 @@ public: /** * Check whether two hashes are equal. */ - bool operator == (const Hash & h2) const; + bool operator == (const Hash & h2) const noexcept; /** * Compare how two hashes are ordered. */ - std::strong_ordering operator <=> (const Hash & h2) const; + std::strong_ordering operator <=> (const Hash & h2) const noexcept; /** * Returns the length of a base-16 representation of this hash. diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9e81132e354..250f9209972 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "error.hh" #include "config.hh" diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index cd5146c8913..012a388c0e7 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -15,11 +15,15 @@ struct MemorySourceAccessor : virtual SourceAccessor * defining what a "file system object" is in Nix. */ struct File { + bool operator == (const File &) const noexcept; + std::strong_ordering operator <=> (const File &) const noexcept; + struct Regular { bool executable = false; std::string contents; - GENERATE_CMP(Regular, me->executable, me->contents); + bool operator == (const Regular &) const = default; + auto operator <=> (const Regular &) const = default; }; struct Directory { @@ -27,13 +31,16 @@ struct MemorySourceAccessor : virtual SourceAccessor std::map> contents; - GENERATE_CMP(Directory, me->contents); + bool operator == (const Directory &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + bool operator < (const Directory &) const noexcept; }; struct Symlink { std::string target; - GENERATE_CMP(Symlink, me->target); + bool operator == (const Symlink &) const = default; + auto operator <=> (const Symlink &) const = default; }; using Raw = std::variant; @@ -41,14 +48,15 @@ struct MemorySourceAccessor : virtual SourceAccessor MAKE_WRAPPER_CONSTRUCTOR(File); - GENERATE_CMP(File, me->raw); - Stat lstat() const; }; File root { File::Directory {} }; - GENERATE_CMP(MemorySourceAccessor, me->root); + bool operator == (const MemorySourceAccessor &) const noexcept = default; + bool operator < (const MemorySourceAccessor & other) const noexcept { + return root < other.root; + } std::string readFile(const CanonPath & path) override; bool pathExists(const CanonPath & path) override; @@ -72,6 +80,20 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; + +inline bool MemorySourceAccessor::File::Directory::operator == ( + const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool MemorySourceAccessor::File::Directory::operator < ( + const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return contents < other.contents; +} + +inline bool MemorySourceAccessor::File::operator == ( + const MemorySourceAccessor::File &) const noexcept = default; +inline std::strong_ordering MemorySourceAccessor::File::operator <=> ( + const MemorySourceAccessor::File &) const noexcept = default; + /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ac2b8353674..04c778c3114 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -148,6 +148,7 @@ sources = files( 'signature/signer.cc', 'source-accessor.cc', 'source-path.cc', + 'strings.cc', 'suggestions.cc', 'tarfile.cc', 'terminal.cc', @@ -215,6 +216,9 @@ headers = [config_h] + files( 'source-accessor.hh', 'source-path.hh', 'split.hh', + 'std-hash.hh', + 'strings.hh', + 'strings-inline.hh', 'suggestions.hh', 'sync.hh', 'tarfile.hh', diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 724e560b7a4..5a2529262ae 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -17,12 +17,6 @@ Pos::operator std::shared_ptr() const return std::make_shared(&*this); } -bool Pos::operator<(const Pos &rhs) const -{ - return std::forward_as_tuple(line, column, origin) - < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); -} - std::optional Pos::getCodeLines() const { if (line == 0) @@ -116,4 +110,50 @@ void Pos::LinesIterator::bump(bool atFirst) input.remove_prefix(eol); } +std::optional Pos::getSnippetUpTo(const Pos & end) const { + assert(this->origin == end.origin); + + if (end.line < this->line) + return std::nullopt; + + if (auto source = getSource()) { + + auto firstLine = LinesIterator(*source); + for (uint32_t i = 1; i < this->line; ++i) { + ++firstLine; + } + + auto lastLine = LinesIterator(*source); + for (uint32_t i = 1; i < end.line; ++i) { + ++lastLine; + } + + LinesIterator linesEnd; + + std::string result; + for (auto i = firstLine; i != linesEnd; ++i) { + auto firstColumn = i == firstLine ? (this->column ? this->column - 1 : 0) : 0; + if (firstColumn > i->size()) + firstColumn = i->size(); + + auto lastColumn = i == lastLine ? (end.column ? end.column - 1 : 0) : std::numeric_limits::max(); + if (lastColumn < firstColumn) + lastColumn = firstColumn; + if (lastColumn > i->size()) + lastColumn = i->size(); + + result += i->substr(firstColumn, lastColumn - firstColumn); + + if (i == lastLine) { + break; + } else { + result += '\n'; + } + } + return result; + } + return std::nullopt; +} + + } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index 9bdf3b4b5db..25217069c14 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -7,6 +7,7 @@ #include #include +#include #include "source-path.hh" @@ -22,21 +23,17 @@ struct Pos struct Stdin { ref source; - bool operator==(const Stdin & rhs) const + bool operator==(const Stdin & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const Stdin & rhs) const - { return *source != *rhs.source; } - bool operator<(const Stdin & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const Stdin & rhs) const noexcept + { return *source <=> *rhs.source; } }; struct String { ref source; - bool operator==(const String & rhs) const + bool operator==(const String & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const String & rhs) const - { return *source != *rhs.source; } - bool operator<(const String & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const String & rhs) const noexcept + { return *source <=> *rhs.source; } }; typedef std::variant Origin; @@ -65,8 +62,16 @@ struct Pos std::optional getCodeLines() const; bool operator==(const Pos & rhs) const = default; - bool operator!=(const Pos & rhs) const = default; - bool operator<(const Pos & rhs) const; + auto operator<=>(const Pos & rhs) const = default; + + std::optional getSnippetUpTo(const Pos & end) const; + + /** + * Get the SourcePath, if the source was loaded from a file. + */ + std::optional getSourcePath() const { + return *std::get_if(&origin); + } struct LinesIterator { using difference_type = size_t; diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 03aa642739c..016fdd74a39 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,9 +1,7 @@ #pragma once ///@file -#include #include -#include #include namespace nix { @@ -89,9 +87,9 @@ public: return p != other.p; } - bool operator < (const ref & other) const + auto operator <=> (const ref & other) const { - return p < other.p; + return p <=> other.p; } private: diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 18f4a79c398..8254a5f899d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -159,13 +159,7 @@ struct FdSource : BufferedSource FdSource(Descriptor fd) : fd(fd) { } FdSource(FdSource &&) = default; - FdSource & operator=(FdSource && s) - { - fd = s.fd; - s.fd = INVALID_DESCRIPTOR; - read = s.read; - return *this; - } + FdSource & operator=(FdSource && s) = default; bool good() override; protected: diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index cc8db01f59c..b16960d4a4c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -4,6 +4,7 @@ #include "canon-path.hh" #include "hash.hh" +#include "ref.hh" namespace nix { @@ -151,9 +152,9 @@ struct SourceAccessor : std::enable_shared_from_this return number == x.number; } - bool operator < (const SourceAccessor & x) const + auto operator <=> (const SourceAccessor & x) const { - return number < x.number; + return number <=> x.number; } void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 023b5ed4b91..759d3c35579 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -47,19 +47,14 @@ SourcePath SourcePath::operator / (const CanonPath & x) const SourcePath SourcePath::operator / (std::string_view c) const { return {accessor, path / c}; } -bool SourcePath::operator==(const SourcePath & x) const +bool SourcePath::operator==(const SourcePath & x) const noexcept { return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); } -bool SourcePath::operator!=(const SourcePath & x) const +std::strong_ordering SourcePath::operator<=>(const SourcePath & x) const noexcept { - return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); -} - -bool SourcePath::operator<(const SourcePath & x) const -{ - return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); + return std::tie(*accessor, path) <=> std::tie(*x.accessor, x.path); } std::ostream & operator<<(std::ostream & str, const SourcePath & path) diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 83ec6295de1..fc2288f747a 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -8,6 +8,7 @@ #include "ref.hh" #include "canon-path.hh" #include "source-accessor.hh" +#include "std-hash.hh" namespace nix { @@ -103,9 +104,8 @@ struct SourcePath */ SourcePath operator / (std::string_view c) const; - bool operator==(const SourcePath & x) const; - bool operator!=(const SourcePath & x) const; - bool operator<(const SourcePath & x) const; + bool operator==(const SourcePath & x) const noexcept; + std::strong_ordering operator<=>(const SourcePath & x) const noexcept; /** * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. @@ -115,8 +115,21 @@ struct SourcePath { return {accessor, accessor->resolveSymlinks(path, mode)}; } + + friend class std::hash; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::SourcePath & s) const noexcept + { + std::size_t hash = 0; + hash_combine(hash, s.accessor->number, s.path); + return hash; + } +}; diff --git a/src/libutil/std-hash.hh b/src/libutil/std-hash.hh new file mode 100644 index 00000000000..c359d11ca63 --- /dev/null +++ b/src/libutil/std-hash.hh @@ -0,0 +1,24 @@ +#pragma once + +//!@file Hashing utilities for use with unordered_map, etc. (ie low level implementation logic, not domain logic like +//! Nix hashing) + +#include + +namespace nix { + +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) {} + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + hash_combine(seed, rest...); +} + +} // namespace nix diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh new file mode 100644 index 00000000000..10c1b19e688 --- /dev/null +++ b/src/libutil/strings-inline.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "strings.hh" + +namespace nix { + +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + bool tail = false; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) { + if (tail) + size += sep.size(); + size += std::string_view(s).size(); + tail = true; + } + std::string s; + s.reserve(size); + tail = false; + for (auto & i : ss) { + if (tail) + s += sep; + s += i; + tail = true; + } + return s; +} + +} // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 00000000000..7ec618bf449 --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,19 @@ +#include + +#include "strings-inline.hh" +#include "util.hh" + +namespace nix { + +template std::string concatStringsSep(std::string_view, const Strings &); +template std::string concatStringsSep(std::string_view, const StringSet &); +template std::string concatStringsSep(std::string_view, const std::vector &); + +typedef std::string_view strings_2[2]; +template std::string concatStringsSep(std::string_view, const strings_2 &); +typedef std::string_view strings_3[3]; +template std::string concatStringsSep(std::string_view, const strings_3 &); +typedef std::string_view strings_4[4]; +template std::string concatStringsSep(std::string_view, const strings_4 &); + +} // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 00000000000..3b112c4096d --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * Concatenate the given strings with a separator between the elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string concatStringsSep(std::string_view, const std::list &); +extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const std::vector &); + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index e67e986fb59..84c8e296f17 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,8 +38,8 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - std::set allMatches, - std::string query) + const std::set & allMatches, + std::string_view query) { std::set res; for (const auto & possibleMatch : allMatches) { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5fad..e39ab400c0d 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "types.hh" #include @@ -20,7 +19,8 @@ public: std::string to_string() const; - GENERATE_CMP(Suggestion, me->distance, me->suggestion) + bool operator ==(const Suggestion &) const = default; + auto operator <=>(const Suggestion &) const = default; }; class Suggestions { @@ -35,8 +35,8 @@ public: ) const; static Suggestions bestMatches ( - std::set allMatches, - std::string query + const std::set & allMatches, + std::string_view query ); Suggestions& operator+=(const Suggestions & other); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f0df527b378..2e323629512 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -174,7 +174,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink) { time_t lastModified = 0; @@ -195,7 +195,12 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin lastModified = std::max(lastModified, archive_entry_mtime(entry)); - switch (archive_entry_filetype(entry)) { + if (auto target = archive_entry_hardlink(entry)) { + parseSink.createHardlink(cpath, CanonPath(target)); + continue; + } + + switch (auto type = archive_entry_filetype(entry)) { case AE_IFDIR: parseSink.createDirectory(cpath); @@ -232,7 +237,7 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin } default: - throw Error("file '%s' in tarball has unsupported file type", path); + throw Error("file '%s' in tarball has unsupported file type %d", path, type); } } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 705d211e437..0517177dbe6 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -41,6 +41,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink); } diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 902e759456d..7ff05a487c3 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -1,7 +1,8 @@ #pragma once ///@file -#include "types.hh" +#include +#include namespace nix { /** diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175d5..325e3ea7351 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,12 +1,10 @@ #pragma once ///@file -#include "ref.hh" #include #include #include -#include #include #include #include diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f4178f87fd9..bcbe9ea4eb2 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -132,7 +132,7 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } -bool ParsedURL::operator ==(const ParsedURL & other) const +bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return scheme == other.scheme diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6cd06e53d17..738ee9f82e6 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -18,7 +18,7 @@ struct ParsedURL std::string to_string() const; - bool operator ==(const ParsedURL & other) const; + bool operator ==(const ParsedURL & other) const noexcept; /** * Remove `.` and `..` path elements. diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 23682ff7ca5..877d1527945 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,8 @@ #include #include +#include "strings.hh" + namespace nix { void initLibUtil(); @@ -33,13 +35,26 @@ template C tokenizeString(std::string_view s, std::string_view separato /** - * Concatenate the given strings with a separator between the - * elements. + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. */ template -std::string concatStringsSep(const std::string_view sep, const C & ss) +[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + // need a cast to string_view since this is also called with Symbols for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); std::string s; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d37b16bdca0..0ce987d8a5c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -183,6 +183,9 @@ static void main_nix_build(int argc, char * * argv) struct MyArgs : LegacyArgs, MixEvalArgs { using LegacyArgs::LegacyArgs; + void setBaseDir(Path baseDir) { + commandBaseDir = baseDir; + } }; MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -286,10 +289,13 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; + if (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) { + myArgs.setBaseDir(absPath(dirOf(script))); + } auto autoArgs = myArgs.getAutoArgs(*state); auto autoArgsWithInNixShell = autoArgs; @@ -334,8 +340,13 @@ static void main_nix_build(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : remainingArgs) { + auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i; + if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); + exprs.push_back(state->parseExprFromString( + std::move(i), + (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".") + )); else { auto absolute = i; try { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c72378cd5f4..5e170c99d37 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index a4bfeebbbdb..c4854951124 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d0840a02e5e..f073074e851 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,13 +2,11 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" -#include "build-result.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" #include "serve-protocol.hh" #include "serve-protocol-connection.hh" -#include "serve-protocol-impl.hh" #include "shared.hh" #include "graphml.hh" #include "legacy.hh" @@ -23,12 +21,14 @@ #include #include -#include #include #include #include +#include "build-result.hh" +#include "exit.hh" +#include "serve-protocol-impl.hh" namespace nix_store { diff --git a/src/nix/.version b/src/nix/.version new file mode 120000 index 00000000000..b7badcd0cc8 --- /dev/null +++ b/src/nix/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/nix/build-remote b/src/nix/build-remote new file mode 120000 index 00000000000..2cea44d46c2 --- /dev/null +++ b/src/nix/build-remote @@ -0,0 +1 @@ +../build-remote \ No newline at end of file diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson new file mode 120000 index 00000000000..91937f18372 --- /dev/null +++ b/src/nix/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 554c3654022..7d9aa771124 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -76,7 +76,9 @@ struct CmdBundle : InstallableValueCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + fetchSettings, bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 9575bf33887..09d14073386 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -1,6 +1,7 @@ #include #include "command.hh" +#include "exit.hh" #include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 552eb94cc0f..e4f6f1e0a9d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -19,6 +19,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct DevelopSettings : Config @@ -698,7 +700,11 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); #endif } }; diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66fbe..46c94b211f9 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -6,6 +6,8 @@ #include +#include "strings.hh" + namespace nix { struct Info diff --git a/src/nix/doc b/src/nix/doc new file mode 120000 index 00000000000..7e57b0f5825 --- /dev/null +++ b/src/nix/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file diff --git a/src/nix/env.cc b/src/nix/env.cc index 021c47cbbd0..9db03ca3780 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,6 +1,10 @@ +#include +#include + #include "command.hh" +#include "eval.hh" #include "run.hh" -#include +#include "strings.hh" using namespace nix; @@ -90,6 +94,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment } } + // TODO: split losslessly; empty means . auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); auto unixPathString = concatStringsSep(":", unixPath); @@ -99,7 +104,11 @@ struct CmdShell : InstallablesCommand, MixEnvironment for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, UseLookupPath::Use, *command.begin(), args); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e197e65bdcf..77b37e9b8fe 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -19,9 +19,10 @@ #include "users.hh" #include -#include #include +#include "strings-inline.hh" + using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -48,19 +49,19 @@ class FlakeCommand : virtual Args, public MixFlakeOptions FlakeRef getFlakeRef() { - return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, absPath(".")); //FIXME } LockedFlake lockFlake() { - return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } std::vector getFlakeRefsForCompletion() override { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), absPath(".")) }; } }; @@ -165,7 +166,7 @@ struct CmdFlakeLock : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); @@ -386,15 +387,15 @@ struct CmdFlakeCheck : FlakeCommand || (hasPrefix(name, "_") && name.substr(1) == expected); }; - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { + auto checkSystemName = [&](std::string_view system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; - auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + auto checkSystemType = [&](std::string_view system, const PosIdx pos) { if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(system); + omittedSystems.insert(std::string(system)); return false; } else { return true; @@ -443,7 +444,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkOverlay = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking overlay '%s'", attrPath)); @@ -462,7 +463,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkModule = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS module '%s'", attrPath)); @@ -473,9 +474,9 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath)); @@ -516,7 +517,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkTemplate = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); @@ -572,7 +573,7 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { + [&](std::string_view name, Value & vOutput, const PosIdx pos) { Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); @@ -596,7 +597,7 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; + std::string_view attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); @@ -796,6 +797,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("some errors were encountered during the evaluation"); if (!omittedSystems.empty()) { + // TODO: empty system is not visible; render all as nix strings? warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", @@ -841,7 +843,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); - auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( + fetchSettings, templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), diff --git a/src/nix/flake.md b/src/nix/flake.md index d69ce7d7ed0..4965019d99f 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -259,6 +259,8 @@ Currently the `type` attribute can be one of the following: `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` can be dropped. + This can also be used to set the location of gitea/forgejo branches. [See here](@docroot@/protocols/tarball-fetcher.md#gitea-and-forgejo-support) + * `file`: Plain files or directory tarballs, either over http(s) or from the local disk. diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 4b0fbb89dfb..d6583449575 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "installable-value.hh" +#include "eval.hh" #include "run.hh" using namespace nix; @@ -49,7 +50,11 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 120000 index 00000000000..5c5624f5ecf --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1 @@ +../../doc/manual/src/store/types/index.md.in \ No newline at end of file diff --git a/src/nix/local.mk b/src/nix/local.mk index 4b61173300a..28b30b58619 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -42,27 +42,16 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/develop.cc: src/nix/get-env.sh.gen.hh +$(d)/develop.cc: $(d)/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: \ +$(d)/main.cc: \ doc/manual/generate-manpage.nix.gen.hh \ doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ doc/manual/generate-store-info.nix.gen.hh \ - src/nix/generated-doc/help-stores.md + $(d)/help-stores.md.gen.hh -src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md - @mkdir -p $$(dirname $@) - @cp $< $@ +$(d)/profile.cc: $(d)/profile.md -src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh - -src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in - @mkdir -p $$(dirname $@) - @echo 'R"(' >> $@.tmp - @echo >> $@.tmp - @cat $^ >> $@.tmp - @echo >> $@.tmp - @echo ')"' >> $@.tmp - @mv $@.tmp $@ +$(d)/profile.md: $(d)/profiles.md.gen.hh diff --git a/src/nix/main.cc b/src/nix/main.cc index de6d89bd371..00ad6fe2c97 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,9 +1,8 @@ -#include - #include "args/root.hh" #include "current-process.hh" #include "command.hh" #include "common-args.hh" +#include "eval-gc.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" @@ -19,6 +18,7 @@ #include "users.hh" #include "network-proxy.hh" #include "eval-cache.hh" +#include "flake/flake.hh" #include #include @@ -41,6 +41,8 @@ extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); #endif +#include "strings.hh" + namespace nix { enum struct AliasStatus { @@ -242,7 +244,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -333,7 +335,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "generated-doc/help-stores.md" + #include "help-stores.md.gen.hh" ; } @@ -362,6 +364,7 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + flake::initLib(flakeSettings); #if __linux__ if (isRootUser()) { @@ -418,7 +421,7 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), evalSettings); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto builtinsJson = nlohmann::json::object(); for (auto & builtin : *state.baseEnv.values[0]->attrs()) { auto b = nlohmann::json::object(); @@ -429,7 +432,7 @@ void mainWrapped(int argc, char * * argv) b["doc"] = trim(stripIndentation(primOp->doc)); if (primOp->experimentalFeature) b["experimental-feature"] = primOp->experimentalFeature; - builtinsJson[state.symbols[builtin.name]] = std::move(b); + builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); } for (auto & [name, info] : state.constantInfos) { auto b = nlohmann::json::object(); diff --git a/src/nix/meson.build b/src/nix/meson.build new file mode 100644 index 00000000000..dd21c4b1bca --- /dev/null +++ b/src/nix/meson.build @@ -0,0 +1,170 @@ +project('nix', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-fetchers'), + dependency('nix-main'), + dependency('nix-cmd'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + #'-include', 'config-fetchers.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/generate-header') + +nix_sources = files( + 'add-to-store.cc', + 'app.cc', + 'build.cc', + 'bundle.cc', + 'cat.cc', + 'config-check.cc', + 'config.cc', + 'copy.cc', + 'derivation-add.cc', + 'derivation-show.cc', + 'derivation.cc', + 'develop.cc', + 'diff-closures.cc', + 'dump-path.cc', + 'edit.cc', + 'env.cc', + 'eval.cc', + 'flake.cc', + 'fmt.cc', + 'hash.cc', + 'log.cc', + 'ls.cc', + 'main.cc', + 'make-content-addressed.cc', + 'nar.cc', + 'optimise-store.cc', + 'path-from-hash-part.cc', + 'path-info.cc', + 'prefetch.cc', + 'profile.cc', + 'realisation.cc', + 'registry.cc', + 'repl.cc', + 'run.cc', + 'search.cc', + 'sigs.cc', + 'store-copy-log.cc', + 'store-delete.cc', + 'store-gc.cc', + 'store-info.cc', + 'store-repair.cc', + 'store.cc', + 'unix/daemon.cc', + 'upgrade-nix.cc', + 'verify.cc', + 'why-depends.cc', +) + +nix_sources += [ + gen_header.process('doc/manual/generate-manpage.nix'), + gen_header.process('doc/manual/generate-settings.nix'), + gen_header.process('doc/manual/generate-store-info.nix'), + gen_header.process('doc/manual/utils.nix'), + gen_header.process('get-env.sh'), + gen_header.process('profiles.md'), + gen_header.process('help-stores.md'), +] + +if host_machine.system() != 'windows' + nix_sources += files( + 'unix/daemon.cc', + ) +endif + +# The rest of the subdirectories aren't separate components, +# just source files in another directory, so we process them here. + +build_remote_sources = files( + 'build-remote/build-remote.cc', +) +nix_build_sources = files( + 'nix-build/nix-build.cc', +) +nix_channel_sources = files( + 'nix-channel/nix-channel.cc', +) +unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') +nix_collect_garbage_sources = files( + 'nix-collect-garbage/nix-collect-garbage.cc', +) +nix_copy_closure_sources = files( + 'nix-copy-closure/nix-copy-closure.cc', +) +nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') +nix_env_sources = files( + 'nix-env/nix-env.cc', + 'nix-env/user-env.cc', +) +nix_instantiate_sources = files( + 'nix-instantiate/nix-instantiate.cc', +) +nix_store_sources = files( + 'nix-store/dotgraph.cc', + 'nix-store/graphml.cc', + 'nix-store/nix-store.cc', +) + +# Hurray for Meson list flattening! +sources = [ + nix_sources, + build_remote_sources, + nix_build_sources, + nix_channel_sources, + unpack_channel_gen, + nix_collect_garbage_sources, + nix_copy_closure_sources, + nix_env_buildenv_gen, + nix_env_sources, + nix_instantiate_sources, + nix_store_sources, +] + +include_dirs = [include_directories('.')] + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) diff --git a/src/nix/nix-build b/src/nix/nix-build new file mode 120000 index 00000000000..2954d8ac7a9 --- /dev/null +++ b/src/nix/nix-build @@ -0,0 +1 @@ +../nix-build \ No newline at end of file diff --git a/src/nix/nix-channel b/src/nix/nix-channel new file mode 120000 index 00000000000..29b75947369 --- /dev/null +++ b/src/nix/nix-channel @@ -0,0 +1 @@ +../nix-channel \ No newline at end of file diff --git a/src/nix/nix-collect-garbage b/src/nix/nix-collect-garbage new file mode 120000 index 00000000000..b037fc1b03f --- /dev/null +++ b/src/nix/nix-collect-garbage @@ -0,0 +1 @@ +../nix-collect-garbage \ No newline at end of file diff --git a/src/nix/nix-copy-closure b/src/nix/nix-copy-closure new file mode 120000 index 00000000000..9063c583ad6 --- /dev/null +++ b/src/nix/nix-copy-closure @@ -0,0 +1 @@ +../nix-copy-closure \ No newline at end of file diff --git a/src/nix/nix-env b/src/nix/nix-env new file mode 120000 index 00000000000..f2f19f58091 --- /dev/null +++ b/src/nix/nix-env @@ -0,0 +1 @@ +../nix-env \ No newline at end of file diff --git a/src/nix/nix-instantiate b/src/nix/nix-instantiate new file mode 120000 index 00000000000..2d7502ffa2e --- /dev/null +++ b/src/nix/nix-instantiate @@ -0,0 +1 @@ +../nix-instantiate \ No newline at end of file diff --git a/src/nix/nix-store b/src/nix/nix-store new file mode 120000 index 00000000000..e6efcac42c3 --- /dev/null +++ b/src/nix/nix-store @@ -0,0 +1 @@ +../nix-store/ \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix new file mode 100644 index 00000000000..fe83c6969d3 --- /dev/null +++ b/src/nix/package.nix @@ -0,0 +1,129 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-expr +, nix-main +, nix-cmd + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + + # Symbolic links to other dirs + ./build-remote + ./doc + ./nix-build + ./nix-channel + ./nix-collect-garbage + ./nix-copy-closure + ./nix-env + ./nix-instantiate + ./nix-store + + # Doc nix files for --help + ../../doc/manual/generate-manpage.nix + ../../doc/manual/utils.nix + ../../doc/manual/generate-settings.nix + ../../doc/manual/generate-store-info.nix + + # Other files to be included as string literals + ../nix-channel/unpack-channel.nix + ../nix-env/buildenv.nix + ./get-env.sh + ./help-stores.md + ../../doc/manual/src/store/types/index.md.in + ./profiles.md + ../../doc/manual/src/command-ref/files/profiles.md + + # Files + ] ++ lib.concatMap + (dir: [ + (fileset.fileFilter (file: file.hasExt "cc") dir) + (fileset.fileFilter (file: file.hasExt "hh") dir) + (fileset.fileFilter (file: file.hasExt "md") dir) + ]) + [ + ./. + ../build-remote + ../nix-build + ../nix-channel + ../nix-collect-garbage + ../nix-copy-closure + ../nix-env + ../nix-instantiate + ../nix-store + ] + ); + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-expr + nix-main + nix-cmd + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 47f9baee505..e7cfb6e7a68 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + using namespace nix; using nlohmann::json; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index c6e60a53be6..db7d9e4efe6 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -195,7 +195,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); + auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c89b8c9bde6..1096f4386be 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -17,6 +17,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct ProfileElementSource @@ -27,7 +29,9 @@ struct ProfileElementSource std::string attrPath; ExtendedOutputsSpec outputs; - bool operator < (const ProfileElementSource & other) const + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const ProfileElementSource & other) const + auto operator < (const ProfileElementSource & other) const { return std::tuple(originalRef.to_string(), attrPath, outputs) < @@ -56,7 +60,7 @@ struct ProfileElement StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); + return dropEmptyInitThenConcatStringsSep(", ", names); } /** @@ -154,8 +158,8 @@ struct ProfileManifest } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), + parseFlakeRef(fetchSettings, e[sOriginalUrl]), + parseFlakeRef(fetchSettings, e[sUrl]), e["attrPath"], e["outputs"].get() }; diff --git a/src/nix/profile.md b/src/nix/profile.md index 9b2f86f4ac6..83a0b5f2991 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "generated-doc/files/profiles.md.gen.hh" +#include "profiles.md.gen.hh" R""( diff --git a/src/nix/profiles.md b/src/nix/profiles.md new file mode 120000 index 00000000000..c67a86194c0 --- /dev/null +++ b/src/nix/profiles.md @@ -0,0 +1 @@ +../../doc/manual/src/command-ref/files/profiles.md \ No newline at end of file diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 8124292406e..ee45162302c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -33,9 +33,9 @@ class RegistryCommand : virtual Args { if (registry) return registry; if (registry_path.empty()) { - registry = fetchers::getUserRegistry(); + registry = fetchers::getUserRegistry(fetchSettings); } else { - registry = fetchers::getCustomRegistry(registry_path); + registry = fetchers::getCustomRegistry(fetchSettings, registry_path); } return registry; } @@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(store); + auto registries = getRegistries(fetchSettings, store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -109,8 +109,8 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand void run() override { - auto fromRef = parseFlakeRef(fromUrl); - auto toRef = parseFlakeRef(toUrl); + auto fromRef = parseFlakeRef(fetchSettings, fromUrl); + auto toRef = parseFlakeRef(fetchSettings, toUrl); auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; @@ -144,7 +144,7 @@ struct CmdRegistryRemove : RegistryCommand, Command void run() override { auto registry = getRegistry(); - registry->remove(parseFlakeRef(url).input); + registry->remove(parseFlakeRef(fetchSettings, url).input); registry->write(getRegistryPath()); } }; @@ -185,8 +185,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { if (locked.empty()) locked = url; auto registry = getRegistry(); - auto ref = parseFlakeRef(url); - auto lockedRef = parseFlakeRef(locked); + auto ref = parseFlakeRef(fetchSettings, url); + auto lockedRef = parseFlakeRef(fetchSettings, locked); registry->remove(ref.input); auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; if (!resolved.isLocked()) diff --git a/src/nix/run.cc b/src/nix/run.cc index c1aae168577..ec6a4d1e82e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, @@ -128,7 +128,11 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + state->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 2fe6ed86ae6..51517fdc94a 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -10,7 +10,7 @@ enum struct UseLookupPath { DontUse }; -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, diff --git a/src/nix/search.cc b/src/nix/search.cc index 97ef1375ed2..7f8504d3f1e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -15,6 +15,8 @@ #include #include +#include "strings.hh" + using namespace nix; using json = nlohmann::json; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 4a7997b1fb0..66d8dbcf092 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -370,9 +370,12 @@ static void daemonLoop(std::optional forceTrustClientOpt) } // Handle the connection. - FdSource from(remote.get()); - FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive); + processConnection( + openUncachedStore(), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + NotRecursive); exit(0); }, options); @@ -437,9 +440,11 @@ static void forwardStdioConnection(RemoteStore & store) { */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - processConnection(store, from, to, trustClient, NotRecursive); + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, NotRecursive); } /** diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 29297253bda..7b3357700e8 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store, evalSettings); + auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2a0cbd19ffb..124a05bed2c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,14 +1,14 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" -#include "references.hh" #include "signals.hh" #include "keys.hh" #include +#include "exit.hh" + using namespace nix; struct CmdVerify : StorePathsCommand diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh index 913fdd9673d..5ca125ab5bc 100644 --- a/tests/functional/characterisation/framework.sh +++ b/tests/functional/characterisation/framework.sh @@ -63,7 +63,7 @@ function characterisationTestExit() { echo '' echo 'You can rerun this test with:' echo '' - echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test" echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index c2c345ed2cb..61af6049a00 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -29,5 +29,26 @@ nix run --no-write-lock-file .#pkgAsPkg ! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" ! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix run -f shell-hello.nix env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expected to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted +# nix run appears to clear _. I don't understand why. Is this ok? +echo "_=..." >> $TEST_ROOT/actual-env +sort $TEST_ROOT/actual-env | uniq > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + clearStore diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 00000000000..82e9940a2e9 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let string = "str"; in [ (/some/path) ((/some/path)) ((/some/path)) ((/some/path + "\n end")) (string) ((string)) ((string)) ((string + "\n end")) ("") ("") ("end") ]) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 100644 index 00000000000..97c9de3cd1d --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1,31 @@ +let + string = "str"; +in [ + /some/path + + ''${/some/path}'' + + '' + ${/some/path}'' + + ''${/some/path} + end'' + + string + + ''${string}'' + + '' + ${string}'' + + ''${string} + end'' + + '''' + + '' + '' + + '' + end'' +] diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 65ff279f868..b9625eb666f 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -65,6 +65,25 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.shebang.expr +chmod a+rx $TEST_ROOT/shell.shebang.expr +# Should fail due to expressions using relative path +! $TEST_ROOT/shell.shebang.expr bar +cp shell.nix config.nix $TEST_ROOT +# Should succeed +echo "cwd: $PWD" +output=$($TEST_ROOT/shell.shebang.expr bar) +[ "$output" = foo ] + +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > $TEST_ROOT/shell.shebang.legacy.expr +chmod a+rx $TEST_ROOT/shell.shebang.legacy.expr +# Should fail due to expressions using relative path +mkdir -p "$TEST_ROOT/somewhere-unrelated" +output="$(cd "$TEST_ROOT/somewhere-unrelated"; $TEST_ROOT/shell.shebang.legacy.expr bar;)" +[[ $(realpath "$output") = $(realpath "$TEST_ROOT/somewhere-unrelated") ]] + # Test nix-shell shebang mode again with metacharacters in the filename. # First word of filename is chosen to not match any file in the test root. sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 86cd6f458d0..4f5bb36aae9 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash source common.sh +source characterisation/framework.sh testDir="$PWD" cd "$TEST_ROOT" @@ -73,21 +74,31 @@ testReplResponseGeneral () { local grepMode commands expectedResponse response grepMode="$1"; shift commands="$1"; shift - expectedResponse="$1"; shift - response="$(nix repl "$@" <<< "$commands" | stripColors)" - echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ - || fail "repl command set: + # Expected response can contain newlines. + # grep can't handle multiline patterns, so replace newlines with TEST_NEWLINE + # in both expectedResponse and response. + # awk ORS always adds a trailing record separator, so we strip it with sed. + expectedResponse="$(printf '%s' "$1" | awk 1 ORS=TEST_NEWLINE | sed 's/TEST_NEWLINE$//')"; shift + # We don't need to strip trailing record separator here, since extra data is ok. + response="$(nix repl "$@" <<< "$commands" 2>&1 | stripColors | awk 1 ORS=TEST_NEWLINE)" + printf '%s' "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ + || fail "$(echo "repl command set: $commands does not respond with: +--- $expectedResponse +--- but with: +--- $response -" +--- + +" | sed 's/TEST_NEWLINE/\n/g')" } testReplResponse () { @@ -189,7 +200,7 @@ testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x ' \ '{ - x = { ... }; + x = «repeated»; y = { ... }; } ' @@ -241,6 +252,45 @@ testReplResponseNoRegex ' ' \ '{ x = «repeated»; - y = { a = 1 }; + y = { a = 1; }; } ' + +# TODO: move init to characterisation/framework.sh +badDiff=0 +badExitCode=0 + +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" + +runRepl () { + + # That is right, we are also filtering out the testdir _without underscores_. + # This is crazy, but without it, GHA will fail to run the tests, showing paths + # _with_ underscores in the set -x log, but _without_ underscores in the + # supposed nix repl output. I have looked in a number of places, but I cannot + # find a mechanism that could cause this to happen. + local testDirNoUnderscores + testDirNoUnderscores="${testDir//_/}" + + # TODO: pass arguments to nix repl; see lang.sh + nix repl 2>&1 \ + | stripColors \ + | sed \ + -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ + -e "s@$nixVersion@@g" \ + -e "s@Added [0-9]* variables@Added variables@g" \ + | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ + ; +} + +for test in $(cd "$testDir/repl"; echo *.in); do + test="$(basename "$test" .in)" + in="$testDir/repl/$test.in" + actual="$testDir/repl/$test.actual" + expected="$testDir/repl/$test.expected" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + diffAndAcceptInner "$test" "$actual" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/repl/characterisation/empty b/tests/functional/repl/characterisation/empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected new file mode 100644 index 00000000000..c10c171e1e8 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. +Added variables. + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:48:5 + + A documented function. + + +"Note that users may not expect this to behave as it currently does" + +Function curriedArgs + … defined at + /path/to/tests/functional/repl/doc-comments.nix:50:5 + + The function returned by applying once + +"This won't produce documentation, because we can't actually document arbitrary values" + +error: value does not have documentation + + diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in new file mode 100644 index 00000000000..8dbbfc37052 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -0,0 +1,7 @@ +:l doc-comments.nix +:doc curriedArgs +x = curriedArgs 1 +"Note that users may not expect this to behave as it currently does" +:doc x +"This won't produce documentation, because we can't actually document arbitrary values" +:doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected new file mode 100644 index 00000000000..704c0050bd8 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. +Added variables. + +"Note that this is not yet complete" + +Function documentedFormals + … defined at + /path/to/tests/functional/repl/doc-comments.nix:57:5 + + Finds x + + diff --git a/tests/functional/repl/doc-comment-formals.in b/tests/functional/repl/doc-comment-formals.in new file mode 100644 index 00000000000..e32fb8ab1a3 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.in @@ -0,0 +1,3 @@ +:l doc-comments.nix +"Note that this is not yet complete" +:doc documentedFormals diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected new file mode 100644 index 00000000000..5ec465a96c9 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.expected @@ -0,0 +1,8 @@ +Nix +Type :? for help. +Function defined at + /path/to/tests/functional/repl/doc-comment-function.nix:2:1 + + A doc comment for a file that only contains a function + + diff --git a/tests/functional/repl/doc-comment-function.in b/tests/functional/repl/doc-comment-function.in new file mode 100644 index 00000000000..8f3c1388a09 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.in @@ -0,0 +1 @@ +:doc import ./doc-comment-function.nix diff --git a/tests/functional/repl/doc-comment-function.nix b/tests/functional/repl/doc-comment-function.nix new file mode 100644 index 00000000000..cdd2413476f --- /dev/null +++ b/tests/functional/repl/doc-comment-function.nix @@ -0,0 +1,3 @@ +/** A doc comment for a file that only contains a function */ +{ ... }: +{ } diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix new file mode 100644 index 00000000000..e91ee0b513d --- /dev/null +++ b/tests/functional/repl/doc-comments.nix @@ -0,0 +1,60 @@ +{ + /** + Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + + ```nix + multiply 2 3 + => 6 + ``` + */ + multiply = x: y: x * y; + + /**👈 precisely this wide 👉*/ + measurement = x: x; + + floatedIn = /** This also works. */ + x: y: x; + + compact=/**boom*/x: x; + + # https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + /** Ignore!!! */ + unambiguous = + /** Very close */ + x: x; + + /** Firmly rigid. */ + constant = true; + + /** Immovably fixed. */ + lib.version = "9000"; + + /** Unchangeably constant. */ + lib.attr.empty = { }; + + lib.attr.undocumented = { }; + + nonStrict = /** My syntax is not strict, but I'm strict anyway. */ x: x; + strict = /** I don't have to be strict, but I am anyway. */ { ... }: null; + # Note that pre and post are the same here. I just had to name them somehow. + strictPre = /** Here's one way to do this */ a@{ ... }: a; + strictPost = /** Here's another way to do this */ { ... }@a: a; + + # TODO + + /** You won't see this. */ + curriedArgs = + /** A documented function. */ + x: + /** The function returned by applying once */ + y: + /** A function body performing summation of two items */ + x + y; + + /** Documented formals (but you won't see this comment) */ + documentedFormals = + /** Finds x */ + { /** The x attribute */ + x + }: x; +} diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected new file mode 100644 index 00000000000..4b05b653c45 --- /dev/null +++ b/tests/functional/repl/doc-compact.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function compact + … defined at + /path/to/tests/functional/repl/doc-comments.nix:18:20 + + boom + + diff --git a/tests/functional/repl/doc-compact.in b/tests/functional/repl/doc-compact.in new file mode 100644 index 00000000000..c87c4e7abbe --- /dev/null +++ b/tests/functional/repl/doc-compact.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc compact diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected new file mode 100644 index 00000000000..c6655833376 --- /dev/null +++ b/tests/functional/repl/doc-constant.expected @@ -0,0 +1,102 @@ +Nix +Type :? for help. +Added variables. + +error: value does not have documentation + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:30:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:33:3 + + Unchangeably constant. + +error: + … while evaluating the attribute 'attr.undocument' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocument' missing + at «string»:1:1: + 1| lib.attr.undocument + | ^ + Did you mean undocumented? + +Attribute constant + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:27:3 + + Firmly rigid. + +Attribute version + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:30:3 + + Immovably fixed. + +Attribute empty + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:33:3 + + Unchangeably constant. + +Attribute undocumented + + … defined at + /path/to/tests/functional/repl/doc-comments.nix:35:3 + + No documentation found. + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing + | ^ + +error: undefined variable 'constanz' + at «string»:1:1: + 1| constanz + | ^ + +error: undefined variable 'missing' + at «string»:1:1: + 1| missing.attr + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing + | ^ + +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing.attr + | ^ + +error: + … while evaluating the attribute 'attr.undocumental' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocumental' missing + at «string»:1:1: + 1| lib.attr.undocumental + | ^ + Did you mean undocumented? + + diff --git a/tests/functional/repl/doc-constant.in b/tests/functional/repl/doc-constant.in new file mode 100644 index 00000000000..9c0dde5e154 --- /dev/null +++ b/tests/functional/repl/doc-constant.in @@ -0,0 +1,15 @@ +:l doc-comments.nix +:doc constant +:doc lib.version +:doc lib.attr.empty +:doc lib.attr.undocument +:doc (import ./doc-comments.nix).constant +:doc (import ./doc-comments.nix).lib.version +:doc (import ./doc-comments.nix).lib.attr.empty +:doc (import ./doc-comments.nix).lib.attr.undocumented +:doc missing +:doc constanz +:doc missing.attr +:doc lib.missing +:doc lib.missing.attr +:doc lib.attr.undocumental diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected new file mode 100644 index 00000000000..30f13572586 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function floatedIn + … defined at + /path/to/tests/functional/repl/doc-comments.nix:16:5 + + This also works. + + diff --git a/tests/functional/repl/doc-floatedIn.in b/tests/functional/repl/doc-floatedIn.in new file mode 100644 index 00000000000..97c12408e63 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc floatedIn diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected new file mode 100644 index 00000000000..43f483ce943 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. +Added variables. + +Function nonStrict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:37:70 + + My syntax is not strict, but I'm strict anyway. + +Function strict + … defined at + /path/to/tests/functional/repl/doc-comments.nix:38:63 + + I don't have to be strict, but I am anyway. + +Function strictPre + … defined at + /path/to/tests/functional/repl/doc-comments.nix:40:48 + + Here's one way to do this + +Function strictPost + … defined at + /path/to/tests/functional/repl/doc-comments.nix:41:53 + + Here's another way to do this + + diff --git a/tests/functional/repl/doc-lambda-flavors.in b/tests/functional/repl/doc-lambda-flavors.in new file mode 100644 index 00000000000..760c99636a6 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.in @@ -0,0 +1,5 @@ +:l doc-comments.nix +:doc nonStrict +:doc strict +:doc strictPre +:doc strictPost diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected new file mode 100644 index 00000000000..8598aaedbc7 --- /dev/null +++ b/tests/functional/repl/doc-measurement.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function measurement + … defined at + /path/to/tests/functional/repl/doc-comments.nix:13:17 + + 👈 precisely this wide 👉 + + diff --git a/tests/functional/repl/doc-measurement.in b/tests/functional/repl/doc-measurement.in new file mode 100644 index 00000000000..fecd5f9d2f0 --- /dev/null +++ b/tests/functional/repl/doc-measurement.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc measurement diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected new file mode 100644 index 00000000000..db82af91fc6 --- /dev/null +++ b/tests/functional/repl/doc-multiply.expected @@ -0,0 +1,15 @@ +Nix +Type :? for help. +Added variables. + +Function multiply + … defined at + /path/to/tests/functional/repl/doc-comments.nix:10:14 + + Perform arithmetic multiplication. It's kind of like + repeated addition, very neat. + + | multiply 2 3 + | => 6 + + diff --git a/tests/functional/repl/doc-multiply.in b/tests/functional/repl/doc-multiply.in new file mode 100644 index 00000000000..bffc6696fd4 --- /dev/null +++ b/tests/functional/repl/doc-multiply.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc multiply diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected new file mode 100644 index 00000000000..825aa1ee170 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. +Added variables. + +Function unambiguous + … defined at + /path/to/tests/functional/repl/doc-comments.nix:24:5 + + Very close + + diff --git a/tests/functional/repl/doc-unambiguous.in b/tests/functional/repl/doc-unambiguous.in new file mode 100644 index 00000000000..8282a5cb95c --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc unambiguous diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected new file mode 100644 index 00000000000..f38b9b56969 --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. +Added variables. + +{ homepage = "https://example.com"; } + +{ homepage = "https://example.com"; } + +{ + layerOne = { ... }; +} + +{ + layerOne = { ... }; +} + +[ "https://example.com" ] + +[ "https://example.com" ] + +[ + [ ... ] +] + +[ + [ ... ] +] + + diff --git a/tests/functional/repl/pretty-print-idempotent.in b/tests/functional/repl/pretty-print-idempotent.in new file mode 100644 index 00000000000..5f865316fdf --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.in @@ -0,0 +1,9 @@ +:l pretty-print-idempotent.nix +oneDeep +oneDeep +twoDeep +twoDeep +oneDeepList +oneDeepList +twoDeepList +twoDeepList diff --git a/tests/functional/repl/pretty-print-idempotent.nix b/tests/functional/repl/pretty-print-idempotent.nix new file mode 100644 index 00000000000..68929f387e4 --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.nix @@ -0,0 +1,19 @@ +{ + oneDeep = { + homepage = "https://" + "example.com"; + }; + twoDeep = { + layerOne = { + homepage = "https://" + "example.com"; + }; + }; + + oneDeepList = [ + ("https://" + "example.com") + ]; + twoDeepList = [ + [ + ("https://" + "example.com") + ] + ]; +} diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index c46fdec8a8c..c920d7cb459 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -55,4 +55,26 @@ rec { chmod +x $out/bin/hello ''; }; + + # execs env from PATH, so that we can probe the environment + # does not allow arguments, because we don't need them + env = mkDerivation { + name = "env"; + outputs = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin + + cat > $out/bin/env <&2 + exit 1 + fi + exec env + EOF + chmod +x $out/bin/env + ''; + }; + } diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 750cdf0bcaa..9cae14b780f 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -46,6 +46,7 @@ let pkgs = rec { ASCII_PERCENT = "%"; ASCII_AT = "@"; TEST_inNixShell = if inNixShell then "true" else "false"; + FOO = fooContents; inherit stdenv; outputs = ["dev" "out"]; } // { diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index b4c03f5477c..c2ac3b24d4a 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -23,6 +23,25 @@ nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' # Test that symlinks outside of the store don't work. expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expectedf to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted +sort $TEST_ROOT/actual-env > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH # https://github.com/NixOS/nix/issues/7905 diff --git a/tests/functional/shell.shebang.expr b/tests/functional/shell.shebang.expr new file mode 100755 index 00000000000..c602dedbf03 --- /dev/null +++ b/tests/functional/shell.shebang.expr @@ -0,0 +1,9 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ./shell.nix" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +echo "$FOO" diff --git a/tests/functional/shell.shebang.legacy.expr b/tests/functional/shell.shebang.legacy.expr new file mode 100755 index 00000000000..490542f430b --- /dev/null +++ b/tests/functional/shell.shebang.legacy.expr @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { fooContents = toString ./.; }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ((builtins.getEnv ''TEST_ROOT'')+''/shell.nix'')" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +#! nix-shell --option nix-shell-shebang-arguments-relative-to-script false +echo "$FOO" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ceb6b13e8ca..4c7e4e50605 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -71,3 +71,15 @@ test_tarball() { test_tarball '' cat test_tarball .xz xz test_tarball .gz gzip + +# Test hard links. +# All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. +# We don't preserve the hard links, because that's an optimization we think is not worth the complexity, +# so we only make sure that the contents are copied correctly. +path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" +[[ $(cat "$path/a/b/foo") = bar ]] +[[ $(cat "$path/a/b/xyzzy") = bar ]] +[[ $(cat "$path/a/yyy") = bar ]] +[[ $(cat "$path/a/zzz") = bar ]] +[[ $(cat "$path/c/aap") = bar ]] +[[ $(cat "$path/fnord") = bar ]] diff --git a/tests/functional/tree.tar.gz b/tests/functional/tree.tar.gz new file mode 100644 index 00000000000..f1f1d996d84 Binary files /dev/null and b/tests/functional/tree.tar.gz differ diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index eacbf0d5cac..54a4a0e9e3b 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -4,12 +4,14 @@ #include #include +#include "fetch-settings.hh" #include "value.hh" #include "nixexpr.hh" +#include "nixexpr.hh" #include "eval.hh" +#include "eval-gc.hh" #include "eval-inline.hh" #include "eval-settings.hh" -#include "store-api.hh" #include "tests/libstore.hh" @@ -24,7 +26,7 @@ namespace nix { protected: LibExprTest() : LibStoreTest() - , state({}, store, evalSettings, nullptr) + , state({}, store, fetchSettings, evalSettings, nullptr) { evalSettings.nixPath = {}; } @@ -43,6 +45,7 @@ namespace nix { } bool readOnlyMode = true; + fetchers::Settings fetchSettings{}; EvalSettings evalSettings{readOnlyMode}; EvalState state; }; diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build index a7b22f7f1e8..ee35258cf27 100644 --- a/tests/unit/libexpr/meson.build +++ b/tests/unit/libexpr/meson.build @@ -41,7 +41,7 @@ add_project_arguments( # It would be nice for our headers to be idempotent instead. '-include', 'config-util.hh', '-include', 'config-store.hh', - '-include', 'config-store.hh', + '-include', 'config-expr.hh', '-include', 'config-util.h', '-include', 'config-store.h', '-include', 'config-expr.h', diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc new file mode 100644 index 00000000000..d3547ec6a31 --- /dev/null +++ b/tests/unit/libfetchers/git-utils.cc @@ -0,0 +1,112 @@ +#include "git-utils.hh" +#include "file-system.hh" +#include "gmock/gmock.h" +#include +#include +#include +#include +#include "fs-sink.hh" +#include "serialise.hh" + +namespace nix { + +class GitUtilsTest : public ::testing::Test +{ + // We use a single repository for all tests. + Path tmpDir; + std::unique_ptr delTmpDir; + +public: + void SetUp() override + { + tmpDir = createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + + // Create the repo with libgit2 + git_libgit2_init(); + git_repository * repo = nullptr; + auto r = git_repository_init(&repo, tmpDir.c_str(), 0); + ASSERT_EQ(r, 0); + git_repository_free(repo); + } + + void TearDown() override + { + // Destroy the AutoDelete, triggering removal + // not AutoDelete::reset(), which would cancel the deletion. + delTmpDir.reset(); + } + + ref openRepo() + { + return GitRepo::openRepo(tmpDir, true, false); + } +}; + +void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable) +{ + if (executable) + fileSink.isExecutable(); + fileSink.preallocateContents(contents.size()); + fileSink(contents); +} + +TEST_F(GitUtilsTest, sink_basic) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory + // here + // The sync method does not document this behavior, should probably renamed because it's not very + // general, and I can't imagine that "non-conventional" archives or any other source to be handled by + // this sink. + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + sink->createRegularFile(CanonPath("foo-1.1/bye"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "thanks for all the fish", false); + }); + sink->createSymlink(CanonPath("foo-1.1/bye-link"), "bye"); + sink->createDirectory(CanonPath("foo-1.1/empty")); + sink->createDirectory(CanonPath("foo-1.1/links")); + sink->createHardlink(CanonPath("foo-1.1/links/foo"), CanonPath("foo-1.1/hello")); + + // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); + + auto result = sink->sync(); + auto accessor = repo->getAccessor(result, false); + auto entries = accessor->readDirectory(CanonPath::root); + ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); + ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); + ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); +}; + +TEST_F(GitUtilsTest, sink_hardlink) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + + try { + sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); + FAIL() << "Expected an exception"; + } catch (const nix::Error & e) { + ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + } +}; + +} // namespace nix diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index 286a590304a..30aa142a5e2 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -29,7 +29,7 @@ libfetchers-tests_LIBS = \ libstore-test-support libutil-test-support \ libfetchers libstore libutil -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) ifdef HOST_WINDOWS # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build index b4bc77a9780..d2de9382933 100644 --- a/tests/unit/libfetchers/meson.build +++ b/tests/unit/libfetchers/meson.build @@ -37,7 +37,7 @@ add_project_arguments( # It would be nice for our headers to be idempotent instead. '-include', 'config-util.hh', '-include', 'config-store.hh', - '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', language : 'cpp', ) diff --git a/tests/unit/libflake/flakeref.cc b/tests/unit/libflake/flakeref.cc index 2b7809b938d..d704a26d36c 100644 --- a/tests/unit/libflake/flakeref.cc +++ b/tests/unit/libflake/flakeref.cc @@ -1,5 +1,6 @@ #include +#include "fetch-settings.hh" #include "flake/flakeref.hh" namespace nix { @@ -11,8 +12,9 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(to_string, doesntReencodeUrl) { + fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; - auto flakeref = parseFlakeRef(s); + auto flakeref = parseFlakeRef(fetchSettings, s); auto parsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/tests/unit/libstore/http-binary-cache-store.cc new file mode 100644 index 00000000000..1e415f6251a --- /dev/null +++ b/tests/unit/libstore/http-binary-cache-store.cc @@ -0,0 +1,21 @@ +#include + +#include "http-binary-cache-store.hh" + +namespace nix { + +TEST(HttpBinaryCacheStore, constructConfig) +{ + HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + + EXPECT_EQ(config.cacheUri, "http://foo.bar.baz"); +} + +TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) +{ + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; + + EXPECT_EQ(config.cacheUri, "https://foo.bar.baz/a/b"); +} + +} // namespace nix diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc new file mode 100644 index 00000000000..eb31a240804 --- /dev/null +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "legacy-ssh-store.hh" + +namespace nix { + +TEST(LegacySSHStore, constructConfig) +{ + LegacySSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/tests/unit/libstore/local-binary-cache-store.cc new file mode 100644 index 00000000000..2e840228dad --- /dev/null +++ b/tests/unit/libstore/local-binary-cache-store.cc @@ -0,0 +1,14 @@ +#include + +#include "local-binary-cache-store.hh" + +namespace nix { + +TEST(LocalBinaryCacheStore, constructConfig) +{ + LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; + + EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc new file mode 100644 index 00000000000..b34ca92375e --- /dev/null +++ b/tests/unit/libstore/local-overlay-store.cc @@ -0,0 +1,34 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-overlay-store.hh" + +namespace nix { + +TEST(LocalOverlayStore, constructConfig_rootQueryParam) +{ + LocalOverlayStoreConfig config{ + "local-overlay", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalOverlayStore, constructConfig_rootPath) +{ + LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc new file mode 100644 index 00000000000..abc3ea7963f --- /dev/null +++ b/tests/unit/libstore/local-store.cc @@ -0,0 +1,40 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-store.hh" + +// Needed for template specialisations. This is not good! When we +// overhaul how store configs work, this should be fixed. +# include "args.hh" +# include "config-impl.hh" +# include "abstract-setting-to-json.hh" + +namespace nix { + +TEST(LocalStore, constructConfig_rootQueryParam) +{ + LocalStoreConfig config{ + "local", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalStore, constructConfig_rootPath) +{ + LocalStoreConfig config{"local", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 90e7d3047c7..8534ba8c52d 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -58,6 +58,11 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'http-binary-cache-store.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', 'machines.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', @@ -66,8 +71,11 @@ sources = files( 'path-info.cc', 'path.cc', 'references.cc', + 's3-binary-cache-store.cc', 'serve-protocol.cc', + 'ssh-store.cc', 'store-reference.cc', + 'uds-remote-store.cc', 'worker-protocol.cc', ) diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 7637cb366aa..9e9c6303d90 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -26,9 +26,9 @@ static UnkeyedValidPathInfo makeEmpty() }; } -static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) +static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) { - UnkeyedValidPathInfo info = ValidPathInfo { + ValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { @@ -57,6 +57,9 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo } return info; } +static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) { + return makeFullKeyed(store, includeImpureInfo); +} #define JSON_TEST(STEM, OBJ, PURE) \ TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ @@ -86,4 +89,13 @@ JSON_TEST(empty_impure, makeEmpty(), true) JSON_TEST(pure, makeFull(*store, false), false) JSON_TEST(impure, makeFull(*store, true), true) +TEST_F(PathInfoTest, PathInfo_full_shortRefs) { + ValidPathInfo it = makeFullKeyed(*store, true); + // it.references = unkeyed.references; + auto refs = it.shortRefs(); + ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); } + +} // namespace nix diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc new file mode 100644 index 00000000000..7aa5f2f2c06 --- /dev/null +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -0,0 +1,18 @@ +#if ENABLE_S3 + +# include + +# include "s3-binary-cache-store.hh" + +namespace nix { + +TEST(S3BinaryCacheStore, constructConfig) +{ + S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + + EXPECT_EQ(config.bucketName, "foobar"); +} + +} // namespace nix + +#endif diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc new file mode 100644 index 00000000000..b853a5f1fb9 --- /dev/null +++ b/tests/unit/libstore/ssh-store.cc @@ -0,0 +1,55 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "ssh-store.hh" + +namespace nix { + +TEST(SSHStore, constructConfig) +{ + SSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +TEST(MountedSSHStore, constructConfig) +{ + MountedSSHStoreConfig config{ + "mounted-ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +} +#endif diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc new file mode 100644 index 00000000000..5ccb208714f --- /dev/null +++ b/tests/unit/libstore/uds-remote-store.cc @@ -0,0 +1,23 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "uds-remote-store.hh" + +namespace nix { + +TEST(UDSRemoteStore, constructConfig) +{ + UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; + + EXPECT_EQ(config.path, "/tmp/socket"); +} + +TEST(UDSRemoteStore, constructConfigWrongScheme) +{ + EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); +} + +} // namespace nix +#endif diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc index 2d0e0dad08f..7a13bd4ff9c 100644 --- a/tests/unit/libutil-support/tests/string_callback.cc +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -2,9 +2,10 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) +void observe_string_cb(const char * start, unsigned int n, void * user_data) { - *user_data = std::string(start); + auto user_data_casted = reinterpret_cast(user_data); + *user_data_casted = std::string(start); } } diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh index a02ea3a1b69..9a7e8d85dab 100644 --- a/tests/unit/libutil-support/tests/string_callback.hh +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -3,14 +3,13 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data); +void observe_string_cb(const char * start, unsigned int n, void * user_data); inline void * observe_string_cb_data(std::string & out) { return (void *) &out; }; -#define OBSERVE_STRING(str) \ - (nix_get_string_callback) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) +#define OBSERVE_STRING(str) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) } diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc new file mode 100644 index 00000000000..122a09dcb32 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -0,0 +1,34 @@ +#include +#include "tracing-file-system-object-sink.hh" + +namespace nix::test { + +void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) +{ + std::cerr << "createDirectory(" << path << ")\n"; + sink.createDirectory(path); +} + +void TracingFileSystemObjectSink::createRegularFile( + const CanonPath & path, std::function fn) +{ + std::cerr << "createRegularFile(" << path << ")\n"; + sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { + // We could wrap this and trace about the chunks of data and such + fn(crf); + }); +} + +void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) +{ + std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; + sink.createSymlink(path, target); +} + +void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) +{ + std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; + sink.createHardlink(path, target); +} + +} // namespace nix::test diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh new file mode 100644 index 00000000000..895ac366405 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -0,0 +1,41 @@ +#pragma once +#include "fs-sink.hh" + +namespace nix::test { + +/** + * A `FileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingFileSystemObjectSink : public virtual FileSystemObjectSink +{ + FileSystemObjectSink & sink; +public: + TracingFileSystemObjectSink(FileSystemObjectSink & sink) + : sink(sink) + { + } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile(const CanonPath & path, std::function fn) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +/** + * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink +{ + ExtendedFileSystemObjectSink & sink; +public: + TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) + : TracingFileSystemObjectSink(sink) + , sink(sink) + { + } + + void createHardlink(const CanonPath & path, const CanonPath & target) override; +}; + +} diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index a0125d023ba..3d01d980609 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -229,7 +229,7 @@ TEST_F(GitTest, both_roundrip) { mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); - ASSERT_EQ(*files, *files2); + ASSERT_EQ(files->root, files2->root); } TEST(GitLsRemote, parseSymrefLineWithReference) { diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc new file mode 100644 index 00000000000..484ecc2479b --- /dev/null +++ b/tests/unit/libutil/position.cc @@ -0,0 +1,122 @@ +#include + +#include "position.hh" + +namespace nix { + +inline Pos::Origin makeStdin(std::string s) +{ + return Pos::Stdin{make_ref(s)}; +} + +TEST(Position, getSnippetUpTo_0) +{ + Pos::Origin o = makeStdin(""); + Pos p(1, 1, o); + ASSERT_EQ(p.getSnippetUpTo(p), ""); +} +TEST(Position, getSnippetUpTo_1) +{ + Pos::Origin o = makeStdin("x"); + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 0, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 99, o); + Pos end(99, 0, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + + // "x" might be preferable, but we only care about not crashing for invalid inputs + ASSERT_EQ(start.getSnippetUpTo(end), ""); + + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + Pos start(1, 1, o); + Pos end(1, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } +} +TEST(Position, getSnippetUpTo_2) +{ + Pos::Origin o = makeStdin("asdf\njkl\nqwer"); + { + Pos start(1, 1, o); + Pos end(1, 2, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "a"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + + // nullopt? I feel like changing the column handling would just make it more fragile + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 2, o); + Pos end(1, 3, o); + ASSERT_EQ(start.getSnippetUpTo(end), "s"); + } + { + Pos start(1, 2, o); + Pos end(2, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\nj"); + } + { + Pos start(1, 2, o); + Pos end(3, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl\nq"); + } + { + Pos start(1, 2, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl"); + } + { + Pos start(1, 4, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "f\njkl"); + } + { + Pos start(1, 5, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); + } + { + Pos start(1, 6, o); // invalid: starting column past last "line character", ie at the newline + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); // jkl might be acceptable for this invalid start position + } + { + Pos start(1, 1, o); + Pos end(2, 0, o); // invalid + ASSERT_EQ(start.getSnippetUpTo(end), "asdf\n"); + } +} + +TEST(Position, example_1) +{ + Pos::Origin o = makeStdin(" unambiguous = \n /** Very close */\n x: x;\n# ok\n"); + Pos start(2, 5, o); + Pos end(2, 22, o); + ASSERT_EQ(start.getSnippetUpTo(end), "/** Very close */"); +} + +} // namespace nix diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc index a517d9aa175..c3efa6d5101 100644 --- a/tests/unit/libutil/references.cc +++ b/tests/unit/libutil/references.cc @@ -15,7 +15,7 @@ struct RewriteParams { strRewrites.insert(from + "->" + to); return os << "OriginalString: " << bar.originalString << std::endl << - "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Rewrites: " << dropEmptyInitThenConcatStringsSep(",", strRewrites) << std::endl << "Expected result: " << bar.finalString; } }; diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc new file mode 100644 index 00000000000..47a20770e2e --- /dev/null +++ b/tests/unit/libutil/strings.cc @@ -0,0 +1,83 @@ +#include + +#include "strings.hh" + +namespace nix { + +using Strings = std::vector; + +/* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(concatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, justOne) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +TEST(concatStringsSep, emptyString) +{ + Strings strings; + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ","); +} + +TEST(concatStringsSep, threeEmptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ",,"); +} + +TEST(concatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); +} + +TEST(concatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); +} + +TEST(concatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 9be4a400d02..8a3ca8561e5 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -227,32 +227,32 @@ namespace nix { } /* ---------------------------------------------------------------------------- - * concatStringsSep + * dropEmptyInitThenConcatStringsSep * --------------------------------------------------------------------------*/ - TEST(concatStringsSep, buildCommaSeparatedString) { + TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); } - TEST(concatStringsSep, buildStringWithEmptySeparator) { + TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { Strings strings; strings.push_back("this"); strings.push_back("is"); strings.push_back("great"); - ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); } - TEST(concatStringsSep, buildSingleString) { + TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { Strings strings; strings.push_back("this"); - ASSERT_EQ(concatStringsSep(",", strings), "this"); + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); } /* ----------------------------------------------------------------------------