diff --git a/.clang-tidy b/.clang-tidy deleted file mode 100644 index 0887b867087a..000000000000 --- a/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -# We use pointers to aggregates in a couple of places, intentionally. -# void * would look weird. -Checks: '-bugprone-sizeof-expression' diff --git a/.clang-tidy b/.clang-tidy new file mode 120000 index 000000000000..871c4b83d57d --- /dev/null +++ b/.clang-tidy @@ -0,0 +1 @@ +nix-meson-build-support/common/clang-tidy/.clang-tidy \ No newline at end of file diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index 2220649caa28..000000000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json -# Disable CodeRabbit auto-review to prevent verbose comments on PRs. -# When enabled: false, CodeRabbit won't attempt reviews and won't post -# "Review skipped" or other automated comments. -reviews: - auto_review: - enabled: false - review_status: false - high_level_summary: false - poem: false - sequence_diagrams: false - changed_files_summary: false - tools: - github-checks: - enabled: false -chat: - art: false - auto_reply: false diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 763c5f27ee6b..000000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,17 +0,0 @@ -# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers. -# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications. -# -# Merge permissions are required for maintaining an entry in this file. -# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ - -# Default reviewers if nothing else matches -* @edolstra - -# This file -.github/CODEOWNERS @edolstra - -# Documentation of built-in functions -src/libexpr/primops.cc @roberth - -# Libstore layer -/src/libstore @ericson2314 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index af94c3e9e5bb..08a5851748d4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,10 +1,9 @@ --- name: Bug report about: Report unexpected or incorrect behaviour -title: '' +title: "" labels: bug -assignees: '' - +assignees: "" --- ## Describe the bug @@ -32,7 +31,9 @@ assignees: '' ## Metadata - + + + ## Additional context @@ -42,13 +43,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [latest Determinate Nix manual] \([source]) - [ ] checked [open bug issues and pull requests] for possible duplicates -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug - ---- - -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index fe9f9dd209d4..b88e10937988 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,10 +1,9 @@ --- name: Feature request about: Suggest a new feature -title: '' +title: "" labels: feature -assignees: '' - +assignees: "" --- ## Is your feature request related to a problem? @@ -27,13 +26,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open feature issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 070e0bd9b25b..430bef971aac 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -1,18 +1,17 @@ --- name: Installer issue about: Report problems with installation -title: '' +title: "" labels: installer -assignees: '' - +assignees: "" --- ## Platform - + -- [ ] Linux: - [ ] macOS +- [ ] Linux: - [ ] WSL ## Additional information @@ -35,13 +34,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open installer issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 4e05b626d398..fcdd0d20135e 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -1,10 +1,9 @@ --- name: Missing or incorrect documentation about: Help us improve the reference manual -title: '' +title: "" labels: documentation -assignees: '' - +assignees: "" --- ## Problem @@ -19,13 +18,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open documentation issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c155bf8bfa4f..d3e1f8177364 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,26 +1,3 @@ - - ## Motivation @@ -34,9 +11,3 @@ PR stuck in review? We have two Nix team meetings per week online that are open - ---- - -Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). - -The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index bc0005413f1a..281d0f79a8b7 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -2,34 +2,21 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. -- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #users:nixos.org](https://matrix.to/#/#users:nixos.org). +- You can always ask for help on [Discord](https://determinate.systems/discord). ## Suggestions for PRs -1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else. -2. If it is unfinished but you plan to finish it, please mark it as a draft. -3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. -4. To get things rolling again, rebase the PR against the target branch and address valid comments. -5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604). -6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions. +1. If it is unfinished but you plan to finish it, please mark it as a draft. +1. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. +1. To get things rolling again, rebase the PR against the target branch and address valid comments. +1. If you need a review to move forward, ask in [Discord](https://determinate.systems/discord). ## Suggestions for issues 1. If it is resolved (either for you personally, or in general), please consider closing it. 2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough. -3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/). -4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude! +3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [Discord](https://determinate.systems/discord). **Memorandum on closing issues** Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort. - -## Useful GitHub search queries - -- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) -- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml deleted file mode 100644 index 535ae9d08fd9..000000000000 --- a/.github/actions/install-nix-action/action.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: "Install Nix" -description: "Helper action for installing Nix with support for dogfooding from master" -inputs: - dogfood: - description: "Whether to use Nix installed from the latest artifact from master branch" - required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" - extra_nix_config: - description: "Gets appended to `/etc/nix/nix.conf` if passed." - install_url: - description: "URL of the Nix installer" - required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false - github_token: - description: "Github token" - required: true - use_cache: - description: "Whether to setup github actions cache (not implemented currently)" - default: false - required: false -runs: - using: "composite" - steps: - - name: "Download nix install artifact from master" - shell: bash - id: download-nix-installer - if: inputs.dogfood == 'true' - run: | - RUN_ID=$(gh run list --repo "$DOGFOOD_REPO" --workflow ci.yml --branch master --status success --json databaseId --jq ".[0].databaseId") - - if [ "$RUNNER_OS" == "Linux" ]; then - INSTALLER_ARTIFACT="installer-linux" - elif [ "$RUNNER_OS" == "macOS" ]; then - INSTALLER_ARTIFACT="installer-darwin" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$INSTALLER_ARTIFACT" - mkdir -p "$INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" - echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" - env: - GH_TOKEN: ${{ inputs.github_token }} - DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} - with: - # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 - install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} - install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} - extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1f2..1880a89da556 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" diff --git a/.github/release-notes.sh b/.github/release-notes.sh new file mode 100755 index 000000000000..f641e146d2e8 --- /dev/null +++ b/.github/release-notes.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# SC2002 disables "useless cat" warnings. +# I prefer pipelines that start with an explicit input, and go from there. +# Overly fussy. +# shellcheck disable=SC2002 + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +finish() { + rm -rf "$scratch" +} +trap finish EXIT + +DATE=$(date +%Y-%m-%d) +DETERMINATE_NIX_VERSION=$(cat .version-determinate) +TAG_NAME="v${DETERMINATE_NIX_VERSION}" +NIX_VERSION=$(cat .version) +NIX_VERSION_MAJOR_MINOR=$(echo "$NIX_VERSION" | cut -d. -f1,2) +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-DeterminateSystems/nix-src}" + +gh api "/repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + -f "tag_name=${TAG_NAME}" > "$scratch/notes.json" + +trim_trailing_newlines() { + local text + text="$(cat)" + echo -n "${text}" +} + +linkify_gh() { + sed \ + -e 's!\(https://github.com/DeterminateSystems/nix-src/\(pull\|issue\)/\([[:digit:]]\+\)\)![DeterminateSystems/nix-src#\3](\1)!' \ + -e 's#\(https://github.com/DeterminateSystems/nix-src/compare/\([^ ]\+\)\)#[\2](\1)#' +} + +( + cat doc/manual/source/release-notes-determinate/changes.md \ + | sed 's/^.*\(\)$/This section lists the differences between upstream Nix '"$NIX_VERSION_MAJOR_MINOR"' and Determinate Nix '"$DETERMINATE_NIX_VERSION"'.\1/' \ + + printf "\n\n" "$DETERMINATE_NIX_VERSION" + cat "$scratch/notes.json" \ + | jq -r .body \ + | grep -v '^#' \ + | grep -v "Full Changelog" \ + | trim_trailing_newlines \ + | sed -e 's/^\* /\n* /' \ + | linkify_gh + echo "" # final newline +) > "$scratch/changes.md" + +( + printf "# Release %s (%s)\n\n" \ + "$DETERMINATE_NIX_VERSION" \ + "$DATE" + printf "* Based on [upstream Nix %s](../release-notes/rl-%s.md).\n\n" \ + "$NIX_VERSION" \ + "$NIX_VERSION_MAJOR_MINOR" + + cat "$scratch/notes.json" | jq -r .body | linkify_gh +) > "$scratch/rl.md" + +( + cat doc/manual/source/SUMMARY.md.in \ + | sed 's/\(\)$/\1\n - [Release '"$DETERMINATE_NIX_VERSION"' ('"$DATE"')](release-notes-determinate\/'"$TAG_NAME"'.md)/' +) > "$scratch/summary.md" + +mv "$scratch/changes.md" doc/manual/source/release-notes-determinate/changes.md +mv "$scratch/rl.md" "doc/manual/source/release-notes-determinate/v${DETERMINATE_NIX_VERSION}.md" +mv "$scratch/summary.md" doc/manual/source/SUMMARY.md.in diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index 7bb3f1c392d5..000000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Backport -on: - pull_request_target: - types: [closed, labeled] -permissions: - contents: read -jobs: - backport: - name: Backport Pull Request - permissions: - # for korthout/backport-action - contents: write - pull-requests: write - if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-24.04-arm - steps: - - name: Generate GitHub App token - id: generate-token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ vars.CI_APP_ID }} - private-key: ${{ secrets.CI_APP_PRIVATE_KEY }} - - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - # required to find all branches - fetch-depth: 0 - - name: Create backport PRs - uses: korthout/backport-action@01619ebc9a6e3f6820274221b9956b3e7365000a # v4.1.0 - id: backport - with: - # Config README: https://github.com/korthout/backport-action#backport-action - github_token: ${{ steps.generate-token.outputs.token }} - github_workspace: ${{ github.workspace }} - auto_merge_enabled: true - pull_description: |- - Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..95d05aa9eac9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,313 @@ +on: + workflow_call: + inputs: + flake: + required: false + default: "./packaging/secure-packages" + type: string + system: + required: true + type: string + runner: + required: true + type: string + runner_for_virt: + required: true + type: string + runner_small: + required: true + type: string + if: + required: false + default: true + type: boolean + run_tests: + required: false + default: true + type: boolean + run_vm_tests: + required: false + default: false + type: boolean + run_regression_tests: + required: false + default: false + type: boolean + publish_manual: + required: false + default: false + type: boolean + upload_artifacts: + required: false + default: true + type: boolean + secrets: + manual_netlify_auth_token: + required: false + manual_netlify_site_id: + required: false + sentry_auth_token: + required: false + sentry_org: + required: false + sentry_project: + required: false + +jobs: + build: + if: ${{ inputs.if }} + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build ${{ inputs.flake }}#packages.${{ inputs.system }}.default .#packages.${{ inputs.system }}.binaryTarball --no-link -L + - run: nix build ${{ inputs.flake }}#packages.${{ inputs.system }}.binaryTarball --out-link tarball + - run: nix build ${{ inputs.flake }}#^debug,out + - name: Upload debug info to Sentry + run: ./maintainers/upload-debug-info-to-sentry.py --debug-dir ./result-debug ./result/bin/nix + if: env.SENTRY_AUTH_TOKEN != '' + env: + SENTRY_AUTH_TOKEN: ${{ secrets.sentry_auth_token }} + SENTRY_ORG: ${{ secrets.sentry_org }} + SENTRY_PROJECT: ${{ secrets.sentry_project }} + - uses: actions/upload-artifact@v6 + if: inputs.upload_artifacts + with: + name: ${{ inputs.system }} + path: ./tarball/*.xz + + build_static: + if: ${{ inputs.if }} + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build ${{ inputs.flake }}#packages.${{ inputs.system }}.nix-cli-static --no-link -L + + test: + if: ${{ inputs.if && inputs.run_tests}} + needs: build + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix flake check --keep-going ${{ inputs.flake }} -L --system ${{ inputs.system }} + + vm_tests_smoke: + if: inputs.run_vm_tests && github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L \ + ${{ inputs.flake }}#hydraJobs.tests.functional_user \ + ${{ inputs.flake }}#hydraJobs.tests.githubFlakes \ + ${{ inputs.flake }}#hydraJobs.tests.nix-docker \ + ${{ inputs.flake }}#hydraJobs.tests.tarballFlakes \ + ; + + vm_tests_all: + if: inputs.run_vm_tests && github.event_name == 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + tests=$(nix flake show --json \ + | jq -r ' + .inventory.hydraJobs.output.children.tests.children + | with_entries(select(.value | has("derivation"))) + | keys[] + | "${{ inputs.flake }}#hydraJobs.tests." + .') + + if [ -z "$tests" ]; then + echo "error: no tests found in hydraJobs.tests" + exit 1 + fi + + cmd() { + nix build -L --keep-going --timeout 600 $tests + } + + if ! cmd; then + echo "failed, retrying once ..." + printf "\n\n\n\n\n\n\n\n" + cmd + fi + + flake_regressions: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + inputs.run_regression_tests + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + strategy: + matrix: + nix_config: + - "lazy-trees = true" + - "lazy-trees = false" + - "eval-cores = 24" + glob: + - "[0-9]*" + - "[a-b]*" + - "[c]*" + - "[d]*" + - "[e]*" + - "[f]*" + - "[g-h]*" + - "[i-k]*" + - "[l]*" + - "[m]*" + - "[n]*" + - "[o-s]*" + - "[t]*" + - "[u-z]*" + + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Run flake regression tests + env: + #PARALLEL: ${{ !contains(matrix.nix_config, 'eval-cores') && '-P 50%' || '-P 1' }} + PARALLEL: '-P 1' + FLAKE_REGRESSION_GLOB: ${{ matrix.glob }} + NIX_CONFIG: ${{ matrix.nix_config }} + PREFETCH: "1" + USE_NIX_FLAKE_SHOW: "1" + run: | + set -x + echo "PARALLEL: $PARALLEL" + echo "NIX_CONFIG: $NIX_CONFIG" + if [ ! -z "${NSC_CACHE_PATH:-}" ]; then + mkdir -p "${NSC_CACHE_PATH}/nix/xdg-cache" + export XDG_CACHE_HOME="${NSC_CACHE_PATH}/nix/xdg-cache" + fi + nix build ${{ inputs.flake }} -L --out-link ./new-nix + export PATH=$(pwd)/new-nix/bin:$PATH + [[ $(type -p nix) = $(pwd)/new-nix/bin/nix ]] + + nix config show lazy-trees + nix config show eval-cores + lscpu + nproc + + if ! flake-regressions/eval-all.sh; then + echo "Some failed, trying again" + printf "\n\n\n\n\n\n\n\n" + NIX_REMOTE=/tmp/nix flake-regressions/eval-all.sh + fi + + manual: + if: github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_small }} + permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + with: + extra-conf: | + extra-experimental-features = provenance + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Build manual + if: inputs.system == 'x86_64-linux' + run: nix build ${{ inputs.flake }}#hydraJobs.manual + - uses: nwtgck/actions-netlify@v3.0 + if: inputs.publish_manual && inputs.system == 'x86_64-linux' + with: + publish-dir: "./result/share/doc/nix/manual" + production-branch: main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but + # its conversation is locked, so this PR comment can never be posted. + # https://github.com/DeterminateSystems/nix-src/pull/165 + enable-pull-request-comment: ${{ github.event.pull_request.number != 165 }} + enable-commit-comment: true + enable-commit-status: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.manual_netlify_auth_token }} + NETLIFY_SITE_ID: ${{ secrets.manual_netlify_site_id }} + + success: + needs: + - build + - test + - vm_tests_smoke + - vm_tests_all + - flake_regressions + - manual + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37193966a25..133d19c2173c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,269 +2,282 @@ name: "CI" on: pull_request: - merge_group: push: branches: - - master - workflow_dispatch: - inputs: - dogfood: - description: 'Use dogfood Nix build' - required: false - default: true - type: boolean - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail + - main + merge_group: + release: + types: + - published -permissions: read-all +permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" jobs: eval: - runs-on: ubuntu-24.04 + runs-on: UbuntuLatest32Cores128G steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: - experimental-features = nix-command flakes - github_token: ${{ secrets.GITHUB_TOKEN }} - use_cache: false - - run: nix flake show --all-systems --json - - pre-commit-checks: - name: pre-commit checks - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/install-nix-action + - uses: actions/checkout@v4 with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: experimental-features = nix-command flakes - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: ./ci/gha/tests/pre-commit-checks + fetch-depth: 0 + - uses: DeterminateSystems/determinate-nix-action@main + - run: nix flake show --all-systems --json + + build_x86_64-linux: + uses: ./.github/workflows/build.yml + # Determinate Secure Packages is a private flake dependency that external + # (fork) contributors can't access, so only build it for non-fork PRs. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + with: + system: x86_64-linux + runner: namespace-profile-linuxamd32c64g-cache + runner_for_virt: UbuntuLatest32Cores128G + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: true + run_regression_tests: true + publish_manual: true + secrets: + manual_netlify_auth_token: ${{ secrets.NETLIFY_AUTH_TOKEN }} + manual_netlify_site_id: ${{ secrets.NETLIFY_SITE_ID }} + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} + + build_x86_64-linux_no_dsp: + uses: ./.github/workflows/build.yml + # Build the non-DSP flake in the merge queue, on tags, and for external (fork) PRs, + # which can't access Determinate Secure Packages. + if: github.event_name == 'merge_group' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'DeterminateSystems/nix-src') + with: + flake: . + system: x86_64-linux + runner: namespace-profile-linuxamd32c64g-cache + runner_for_virt: UbuntuLatest32Cores128G + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: false + run_regression_tests: false + upload_artifacts: false + secrets: + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} - basic-checks: - name: aggregate basic checks + build_aarch64-linux: + uses: ./.github/workflows/build.yml + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + with: + system: aarch64-linux + runner: UbuntuLatest32Cores128GArm + runner_for_virt: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm + secrets: + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} + + build_aarch64-linux_no_dsp: + uses: ./.github/workflows/build.yml + if: github.event_name == 'merge_group' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'DeterminateSystems/nix-src') + with: + flake: . + system: aarch64-linux + runner: UbuntuLatest32Cores128GArm + runner_for_virt: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm + upload_artifacts: false + secrets: + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} + + build_aarch64-darwin: + uses: ./.github/workflows/build.yml + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + with: + system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_for_virt: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge + secrets: + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} + + build_aarch64-darwin_no_dsp: + uses: ./.github/workflows/build.yml + if: github.event_name == 'merge_group' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'DeterminateSystems/nix-src') + with: + flake: . + system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_for_virt: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge + upload_artifacts: false + secrets: + sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }} + sentry_org: ${{ secrets.SENTRY_ORG }} + sentry_project: ${{ secrets.SENTRY_PROJECT }} + + success: + runs-on: ubuntu-latest + needs: + - eval + - build_x86_64-linux + - build_aarch64-linux + - build_aarch64-darwin + - build_x86_64-linux_no_dsp + - build_aarch64-linux_no_dsp + - build_aarch64-darwin_no_dsp if: ${{ always() }} - runs-on: ubuntu-24.04 - needs: [pre-commit-checks, eval] steps: - - name: Exit with any errors - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - run: | + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') - tests: - needs: basic-checks - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - instrumented: false - primary: true - stdenv: stdenv - - scenario: on macos - runs-on: macos-14 - os: darwin - instrumented: false - primary: true - stdenv: stdenv - - scenario: on ubuntu (with sanitizers / coverage) - runs-on: ubuntu-24.04 - os: linux - instrumented: true - primary: false - stdenv: clangStdenv - name: tests ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 60 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" - # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: - # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'linux' - - name: Run component tests - run: | - nix build --file ci/gha/tests/wrapper.nix componentTests -L \ - --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" - - name: Run VM tests - run: | - nix build --file ci/gha/tests/wrapper.nix vmTests -L \ - --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" - if: ${{ matrix.os == 'linux' }} - - name: Run flake checks and prepare the installer tarball - run: | - ci/gha/tests/build-checks - ci/gha/tests/prepare-installer-for-github-actions - if: ${{ matrix.primary }} - - name: Collect code coverage - run: | - nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \ - --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" \ - --out-link coverage-reports - cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY - if: ${{ matrix.instrumented }} - - name: Upload coverage reports - uses: actions/upload-artifact@v6 - with: - name: coverage-reports - path: coverage-reports/ - if: ${{ matrix.instrumented }} - - name: Upload installer tarball - uses: actions/upload-artifact@v6 - with: - name: installer-${{matrix.os}} - path: out/* - if: ${{ matrix.primary }} + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main - windows_tests: - needs: basic-checks - name: windows unit tests - runs-on: ubuntu-24.04 - continue-on-error: true - timeout-minutes: 60 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - - name: Run Windows unit tests - run: | - nix build --file ci/gha/tests/windows.nix unitTests.nix-util-tests -L + - name: Create artifacts directory + run: mkdir -p ./artifacts - installer_test: - needs: [tests] - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - experimental-installer: false - - scenario: on macos - runs-on: macos-14 - os: darwin - experimental-installer: false - - scenario: on ubuntu (experimental) - runs-on: ubuntu-24.04 - os: linux - experimental-installer: true - - scenario: on macos (experimental) - runs-on: macos-14 - os: darwin - experimental-installer: true - name: installer test ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v6 - - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: installer-${{matrix.os}} - path: out - - name: Looking up the installer tarball URL - id: installer-tarball-url - run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@2126ae7fc54c9df00dd18f7f18754393182c73cd # v31.9.1 - if: ${{ !matrix.experimental-installer }} - with: - install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} - install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - uses: ./.github/actions/install-nix-action - if: ${{ matrix.experimental-installer }} - with: - dogfood: false - experimental-installer: true - tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: sudo apt install fish zsh - if: matrix.os == 'linux' - - run: brew install fish - if: matrix.os == 'darwin' - - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + - name: Fetch artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: downloaded + - name: Move downloaded artifacts to artifacts directory + run: | + shopt -s nullglob + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done - flake_regressions: - needs: tests - runs-on: ubuntu-24.04 - steps: - - name: Checkout nix - uses: actions/checkout@v6 - - name: Checkout flake-regressions - uses: actions/checkout@v6 + - name: Build fallback-paths.nix + id: build + if: ${{ + github.event_name != 'pull_request' + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + ) + }} + env: + INCLUDE_NO_DSP: ${{ + needs.build_x86_64-linux_no_dsp.result == 'success' + && needs.build_aarch64-linux_no_dsp.result == 'success' + && needs.build_aarch64-darwin_no_dsp.result == 'success' + }} + run: | + for system in x86_64-linux aarch64-linux aarch64-darwin; do + nix build ./packaging/secure-packages#packages."$system".default^out,dev,doc,man --out-link /tmp/outputs/"$system"-dsp -j0 + [[ "$INCLUDE_NO_DSP" == "true" ]] && nix build .#packages."$system".default^out,dev,doc,man --out-link /tmp/outputs/"$system"-non-dsp -j0 + done + + [[ "$INCLUDE_NO_DSP" == "true" ]] && nix build .#fallbackPathsNix --out-link /tmp/outputs/fallback-non-dsp + nix build ./packaging/secure-packages#fallbackPathsNix --out-link /tmp/outputs/fallback-dsp + cat /tmp/outputs/fallback-dsp > ./artifacts/fallback-paths.nix + + outputs=() + for path in /tmp/outputs/*; do + outputs+=("$(readlink $path)") + done + echo "nix-outputs=${outputs[@]}" >> "$GITHUB_OUTPUT" + + - uses: DeterminateSystems/public-slice-mirror/.github/actions/upload-store-paths@main with: - repository: NixOS/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v6 + store-paths: | + ${{ steps.build.outputs.nix-outputs }} + + - uses: DeterminateSystems/push-artifact-ids@main with: - repository: NixOS/flake-regressions-data - path: flake-regressions/tests - - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true + allowed_branches: '["main"]' + outputs: + nix-outputs: ${{ steps.build.outputs.nix-outputs }} + + publish: + needs: + - success + # NOTE(cole-h): we need always() here because if any of the needs jobs are skipped (this is + # transitive as well, so if any of success's needs jobs are skipped, like the *_no_dsp ones + # usually are on release), this gets skipped too... + if: (always() && github.repository == 'DeterminateSystems/nix-src' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) + environment: ${{ github.event_name == 'release' && 'production' || '' }} + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-push@main with: - name: installer-linux - path: out - - name: Looking up the installer tarball URL - id: installer-tarball-url + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + visibility: "public" + tag: "${{ github.ref_name }}" + include-output-paths: true + - name: Ensure that we actually released + if: startsWith(github.ref, 'refs/tags/') + env: + TAG_NAME: ${{ github.ref_name }} run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@2126ae7fc54c9df00dd18f7f18754393182c73cd # v31.9.1 - with: - install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} - install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - name: Run flake regressions tests - run: MAX_FLAKES=25 flake-regressions/eval-all.sh + nix flake metadata https://flakehub.com/f/DeterminateSystems/nix-src/"$TAG_NAME" + - name: Update the release notes + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ github.ref_name }} + run: | + gh release edit "$TAG_NAME" --notes-file doc/manual/source/release-notes-determinate/"$TAG_NAME".md || true - profile_build: - needs: tests - runs-on: ubuntu-24.04 - timeout-minutes: 60 - if: >- - github.event_name == 'push' && - github.ref_name == 'master' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: | - experimental-features = flakes nix-command ca-derivations impure-derivations - max-jobs = 1 - - run: | - nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md - cat build-time-report.md >> $GITHUB_STEP_SUMMARY + mirror: + needs: success + permissions: + id-token: write # required so the workflow can auth to FlakeHub + contents: read + if: ${{ + always() + && ( + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + || startsWith(github.ref, 'refs/tags/') + || ( + github.event_name == 'pull_request' + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + ) + ) + }} + uses: DeterminateSystems/public-slice-mirror/.github/workflows/subset-mirror.yml@main + with: + flake-name: DeterminateSystems/nix-artifacts + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + tag: "${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || '' }}" + checkout-ref: "${{ github.ref }}" + flakehub-push: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/') }} diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml deleted file mode 100644 index 16038cb213ca..000000000000 --- a/.github/workflows/labels.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: "Label PR" - -on: - pull_request_target: - types: [edited, opened, synchronize, reopened] - -# WARNING: -# When extending this action, be aware that $GITHUB_TOKEN allows some write -# access to the GitHub API. This means that it should not evaluate user input in -# a way that allows code injection. - -permissions: - contents: read - pull-requests: write - -jobs: - labels: - runs-on: ubuntu-24.04 - if: github.repository_owner == 'NixOS' - steps: - - uses: actions/labeler@v6 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: false diff --git a/.github/workflows/propose-release.yml b/.github/workflows/propose-release.yml new file mode 100644 index 000000000000..7593dc9e8510 --- /dev/null +++ b/.github/workflows/propose-release.yml @@ -0,0 +1,33 @@ +on: + workflow_dispatch: + inputs: + reference-id: + type: string + required: true + version: + type: string + required: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + propose-release: + uses: DeterminateSystems/propose-release/.github/workflows/workflow.yml@main + permissions: + id-token: write + contents: write + pull-requests: write + with: + update-flake: false + reference-id: ${{ inputs.reference-id }} + version: ${{ inputs.version }} + extra-commands-early: | + nix flake update --commit-lock-file --flake ./packaging/secure-packages + echo ${{ inputs.version }} > .version-determinate + git add .version-determinate + git commit -m "Set .version-determinate to ${{ inputs.version }}" || true + ./.github/release-notes.sh + git add doc + git commit -m "Generate release notes for ${{ inputs.version }}" || true diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml deleted file mode 100644 index 5de9ec7951ce..000000000000 --- a/.github/workflows/upload-release.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Upload Release -on: - workflow_dispatch: - inputs: - eval_id: - description: "Hydra evaluation ID" - required: true - type: number - is_latest: - description: "Mark as latest release" - required: false - type: boolean - default: false -permissions: - contents: read - id-token: write - packages: write -jobs: - release: - runs-on: ubuntu-24.04 - environment: releases - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: ./.github/actions/install-nix-action - with: - dogfood: false # Use stable version - use_cache: false # Don't want any cache injection shenanigans - extra_nix_config: | - experimental-features = nix-command flakes - - name: Set NIX_PATH from flake input - run: | - NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link) - # Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe - # get the same uberhack that nix-shell has to support it. - echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 - with: - role-to-assume: "arn:aws:iam::080433136561:role/nix-release" - role-session-name: nix-release-oidc-${{ github.run_id }} - aws-region: eu-west-1 - - name: Disable containerd image store - run: | - # Docker 28+ defaults to the containerd image store, which - # pushes layers uncompressed instead of gzip. OCI clients - # that only support gzip (e.g. go-containerregistry) fail - # with "gzip: invalid header". Disabling the containerd - # snapshotter restores the classic storage driver, which - # preserves gzip-compressed layers through the - # `docker load` / `docker push` pipeline. - echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null - sudo systemctl restart docker - - name: Login to Docker Hub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Upload release - run: | - ./maintainers/upload-release.pl \ - ${{ inputs.eval_id }} \ - --skip-git - env: - IS_LATEST: ${{ inputs.is_latest && '1' || '' }} - - name: Push to GHCR - run: | - DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix" - ./maintainers/upload-release.pl \ - ${{ inputs.eval_id }} \ - --skip-git \ - --skip-s3 \ - --docker-owner "$DOCKER_OWNER" - env: - IS_LATEST: ${{ inputs.is_latest && '1' || '' }} diff --git a/.gitignore b/.gitignore index 96c134335232..c0d09b7e189c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,23 +2,8 @@ /build # Meson creates this file too src/.wraplock - -# /tests/functional/ -/tests/functional/common/subst-vars.sh -/tests/functional/restricted-innocent -/tests/functional/debugger-test-out -/tests/functional/test-libstoreconsumer/test-libstoreconsumer -/tests/functional/nix-shell - -# /tests/functional/lang/ -/tests/functional/lang/*.out -/tests/functional/lang/*.out.xml -/tests/functional/lang/*.err -/tests/functional/lang/*.ast - -# /tests/functional/cli-characterisation/ -/tests/functional/cli-characterisation/*.out -/tests/functional/cli-characterisation/*.err +# Python bytecode (clang-tidy runner scripts) +__pycache__/ /outputs diff --git a/.version b/.version index 714469a714d3..d07233cc933c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.34.6 +2.35.1 diff --git a/.version-determinate b/.version-determinate new file mode 100644 index 000000000000..ac3a4bd51e63 --- /dev/null +++ b/.version-determinate @@ -0,0 +1 @@ +3.21.9 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..27268bf11f5f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +Observe the following rules when contributing to this repository: + +* Before committing, run ./maintainers/format.sh to detect/fix any formatting issues. + +* Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits. diff --git a/README.md b/README.md index 02498944cdb7..c5cbcbed21bb 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,111 @@ -# Nix +

+ +

+

+  Discord  +  Bluesky  +  Mastodon  +  Twitter  +  LinkedIn  +

-[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) -[![CI](https://github.com/NixOS/nix/workflows/CI/badge.svg)](https://github.com/NixOS/nix/actions/workflows/ci.yml) +# The Determinate Nix CLI -Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) -for more details. +[![CI](https://github.com/DeterminateSystems/nix-src/workflows/CI/badge.svg)](https://github.com/DeterminateSystems/nix-src/actions/workflows/ci.yml) -## Installation and first steps +**Nix** is a powerful [language], [package manager][package-management], and [build tool][cli] for [macOS](#macos), [Linux](#linux), and other Unix systems. +It enables you to create fully reproducible [development environments][envs], to build [packages] in sandboxed environments, to build entire Linux systems using [NixOS], and much more. -Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). +[**Determinate Nix**][det-nix] is a downstream distribution of [Nix][upstream] created and maintained by [Determinate Systems][detsys]. +It has two components: -Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). +- The Determinate Nix CLI, a distribution of the Nix CLI built from this repository. + It's based on the [upstream Nix CLI][upstream] and continuously rebased against it, but adds a wide variety of [features] and [improvements][changelog]. +- [Determinate Nixd][dnixd] is a useful daemon for Linux and macOS that handles vital tasks like configuration and enterprise certificate management. -## Building and developing +Determinate Nix is built on SOC-2-Type-II-compliant infrastructure using [Determinate Secure Packages][secure-packages], released via a carefully orchestrated process, and, for Determinate Systems customers, backed by formal security response SLAs that meet stringent compliance standards. -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). +> [!NOTE] +> Determinate Nix, by definition, consists of _both_ the components listed above. +> While it's possible to use the code in this repository to run just our downstream Nix CLI, we do _not_ officially support this experience and provide none of the guarantees or SLAs that we provide for Determinate Nix proper. -## Contributing +Determinate Nix is part of the [Determinate platform][determinate], which also includes [FlakeHub], a secure flake repository with features like [FlakeHub Cache][cache], [private flakes][private-flakes], and [semantic versioning][semver] (SemVer) for [flakes]. + +## Installing Determinate Nix + +You can install Determinate Nix on [macOS](#macos), non-NixOS [Linux](#linux) and WSL, and [NixOS](#nixos). + +### macOS + +On macOS, we recommend using the graphical installer from Determinate Systems. +Click [here][gui] to download and run it. + +### Linux + +On Linux, including Windows Subsystem for Linux (WSL), we recommend installing Determinate Nix using [Determinate Nix Installer][installer]: + +```shell +curl -fsSL https://install.determinate.systems/nix | sh -s -- install +``` -Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. +### NixOS -## Additional resources +On [NixOS], we recommend following our [dedicated installation guide][nixos-install]. +We also provide both [Amazon Machine Images][amis] (AMIs) and [ISOs] for using Determinate on NixOS. -Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. -Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. +## Other resources -- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) -- [Official documentation on nix.dev](https://nix.dev) -- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) -- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively -- [Discourse](https://discourse.nixos.org/) -- Matrix: [#users:nixos.org](https://matrix.to/#/#users:nixos.org) for user support and [#nix-dev:nixos.org](https://matrix.to/#/#nix-dev:nixos.org) for development +Nix was created by [Eelco Dolstra][eelco] and developed as the subject of his 2006 PhD thesis, [The Purely Functional Software Deployment Model][thesis]. +Today, a worldwide developer community contributes to Nix and the ecosystem that has grown around it. + +- [Zero to Nix][z2n], Determinate Systems' guide to Nix and [flakes] for beginners +- [Nixpkgs], a collection of well over 100,000 software packages that you can build and manage using Nix +- [NixOS] is a Linux distribution that can be configured fully declaratively +- The Nix, Nixpkgs, and NixOS community on [nixos.org][website] + +## Reference + +The primary documentation for Determinate and Determinate Nix is available at [docs.determinate.systems][determinate]. +For deeply technical reference material, see the [Determinate Nix manual][manual] which is based on the upstream Nix manual. ## License -Nix is released under the [LGPL v2.1](./COPYING). +[Upstream Nix][upstream] is released under the [LGPL v2.1][license] license. +[Determinate Nix][det-nix] is also released under LGPL v2.1 in accordance with the terms of the upstream license. + +## Contributing + +Check the [contributing guide][contributing] if you want to get involved with developing Nix. + +[amis]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html +[cache]: https://docs.determinate.systems/flakehub/cache +[changelog]: https://determinate.systems/blog/categories/changelog +[cli]: https://manual.determinate.systems/command-ref/new-cli/nix.html +[contributing]: ./CONTRIBUTING.md +[det-nix]: https://docs.determinate.systems/determinate-nix +[determinate]: https://docs.determinate.systems +[detsys]: https://determinate.systems +[dnixd]: https://docs.determinate.systems/determinate-nix#determinate-nixd +[eelco]: https://determinate.systems/people/eelco-dolstra +[envs]: https://zero-to-nix.com/concepts/dev-env +[features]: https://docs.determinate.systems/determinate-nix/#special-features +[flakehub]: https://flakehub.com +[flakes]: https://zero-to-nix.com/concepts/flakes +[gui]: https://install.determinate.systems/determinate-pkg/stable/Universal +[installer]: https://github.com/DeterminateSystems/nix-installer +[isos]: https://github.com/DeterminateSystems/nixos-iso +[language]: https://zero-to-nix.com/concepts/nix-language +[license]: ./COPYING +[manual]: https://manual.determinate.systems +[nixpkgs]: https://github.com/NixOS/nixpkgs +[nixos]: https://github.com/NixOS/nixpkgs/tree/master/nixos +[nixos-install]: https://docs.determinate.systems/guides/advanced-installation#nixos +[packages]: https://zero-to-nix.com/concepts/packages +[package-management]: https://zero-to-nix.com/concepts/package-management +[private-flakes]: https://docs.determinate.systems/flakehub/private-flakes +[secure-packages]: https://determinate.systems/secure-packages +[semver]: https://docs.determinate.systems/flakehub/concepts/semver +[thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[upstream]: https://github.com/NixOS/nix +[website]: https://nixos.org +[z2n]: https://zero-to-nix.com diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 6100f2f4172e..94327adc9f0d 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); @@ -76,13 +74,19 @@ rec { */ topLevel = { installerScriptForGHA = hydraJobs.installerScriptForGHA.${system}; - installTests = hydraJobs.installTests.${system}; nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system}; + nixpkgsLibTestsLazy = hydraJobs.tests.nixpkgsLibTestsLazy.${system}; + filetransfer-retry-backoff = hydraJobs.tests.filetransfer-retry-backoff.${system}; rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out ''; repl-completion = pkgs.callPackage ../../../tests/repl-completion.nix { inherit (packages') nix; }; + lazyTrees = nixComponents.nix-functional-tests.override { + pname = "nix-lazy-trees-tests"; + lazyTrees = true; + }; + /** Checks for our packaging expressions. This shouldn't build anything significant; just check that things diff --git a/ci/gha/tests/prepare-installer-for-github-actions b/ci/gha/tests/prepare-installer-for-github-actions index 0fbecf25c2aa..e240e56fca0a 100755 --- a/ci/gha/tests/prepare-installer-for-github-actions +++ b/ci/gha/tests/prepare-installer-for-github-actions @@ -2,10 +2,14 @@ set -euo pipefail -nix build -L ".#installerScriptForGHA" ".#binaryTarball" +nix build -L \ + ".#installerScriptForGHA" \ + ".#binaryTarball" \ + ".#rustInstaller" mkdir -p out cp ./result/install "out/install" name="$(basename "$(realpath ./result-1)")" # everything before the first dash cp -r ./result-1 "out/${name%%-*}" +cp ./result-2/bin/nix-installer "out/nix-installer" diff --git a/default.nix b/default.nix deleted file mode 100644 index 6466507b7140..000000000000 --- a/default.nix +++ /dev/null @@ -1,9 +0,0 @@ -(import ( - let - lock = builtins.fromJSON (builtins.readFile ./flake.lock); - in - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; - sha256 = lock.nodes.flake-compat.locked.narHash; - } -) { src = ./.; }).defaultNix diff --git a/doc/manual/book.toml.in b/doc/manual/book.toml.in index c798afc4a8c0..11efca75f110 100644 --- a/doc/manual/book.toml.in +++ b/doc/manual/book.toml.in @@ -1,12 +1,12 @@ [book] -title = "Nix @version@ Reference Manual" +title = "Determinate Nix @version@ Reference Manual" src = "source" [output.html] additional-css = ["custom.css"] additional-js = ["redirects.js"] -edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" -git-repository-url = "https://github.com/NixOS/nix" +edit-url-template = "https://github.com/DeterminateSystems/nix-src/tree/master/doc/manual/{path}" +git-repository-url = "https://github.com/DeterminateSystems/nix-src" mathjax-support = true # Handles replacing @docroot@ with a path to ./source relative to that markdown file, diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 7af150be391b..119c6d125430 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -1,5 +1,5 @@ :root { - --sidebar-width: 23em; + --sidebar-width: 23em; } h1.menu-title::before { @@ -7,11 +7,10 @@ h1.menu-title::before { background-image: url("./favicon.svg"); padding: 1.25em; background-position: center center; - background-size: 2em; + background-size: 1.5em; background-repeat: no-repeat; } - .menu-bar { padding: 0.5em 0em; } @@ -21,13 +20,13 @@ h1.menu-title::before { } h1:not(:first-of-type) { - margin-top: 1.3em; + margin-top: 1.3em; } h2 { - margin-top: 1em; + margin-top: 1em; } .hljs-meta { - user-select: none; + user-select: none; } diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 31e74e17d264..7b2a02bd3f84 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -42,11 +42,6 @@ let let result = '' - > **Warning** \ - > This program is - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and its interface is subject to change. - # Name `${command}` - ${details.description} @@ -81,7 +76,11 @@ let subcommands = if length categories > 1 then listCategories else listSubcommands details.commands; categories = sort (x: y: x.id < y.id) ( - unique (map (cmd: cmd.category) (attrValues details.commands)) + unique ( + map (cmd: { inherit (cmd.category) id description; }) ( + builtins.filter (cmd: cmd.category.id != 103) (attrValues details.commands) + ) + ) ); listCategories = concatStrings (map showCategory categories); @@ -89,7 +88,7 @@ let showCategory = cat: '' **${toString cat.description}:** - ${listSubcommands (filterAttrs (n: v: v.category == cat) details.commands)} + ${listSubcommands (filterAttrs (n: v: v.category.id == cat.id) details.commands)} ''; listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds)); diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e66611affe08..a58d4f79e132 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -46,10 +46,12 @@ let ## Settings - ${showSettings { - prefix = "store-${slug}"; - inherit inlineHTML; - } settings} + ${replaceStrings [ "@store-slug@" ] [ "store-${slug}" ] ( + showSettings { + prefix = "store-${slug}"; + inherit inlineHTML; + } settings + )} ''; experimentalFeatureNote = optionalString (experimentalFeature != null) '' diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fd841e80cbb..1edf11afc1f4 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,23 +1,13 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) -# Compute documentation URL based on version and release type -version = meson.project_version() -official_release = get_option('official-release') +fs = import('fs') -if official_release - # For official releases, use versioned URL (dropping patch version) - version_parts = version.split('.') - major_minor = '@0@.@1@'.format(version_parts[0], version_parts[1]) - doc_url = 'https://nix.dev/manual/nix/@0@'.format(major_minor) -else - # For development builds, use /latest - doc_url = 'https://nix.dev/manual/nix/latest' -endif +doc_url = 'https://manual.determinate.systems/' nix = find_program('nix', native : true) @@ -39,7 +29,7 @@ nix_env_for_docs = { 'NIX_CONFIG' : 'cores = 0', } -nix_for_docs = [ nix, '--experimental-features', 'nix-command' ] +nix_for_docs = [ nix ] nix_eval_for_docs_common = nix_for_docs + [ 'eval', '-I', @@ -141,7 +131,7 @@ if get_option('html-manual') python.full_path(), mdbook.full_path(), meson.current_build_dir(), - meson.project_version(), + fs.read('../../.version-determinate').strip(), ), ], input : [ @@ -241,7 +231,9 @@ foreach command : nix_nested_manpages meson.current_source_dir() / 'source', meson.current_build_dir() / 'source', doc_url, - meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'), + meson.current_source_dir() / 'source/command-ref' / command[0] / ( + page + '.md' + ), '@OUTPUT0@', ], input : [ @@ -257,6 +249,7 @@ foreach command : nix_nested_manpages endforeach nix3_manpages = [ + 'nix', 'nix3-build', 'nix3-bundle', 'nix3-config', @@ -264,29 +257,34 @@ nix3_manpages = [ 'nix3-config-show', 'nix3-copy', 'nix3-daemon', - 'nix3-derivation-add', 'nix3-derivation', + 'nix3-derivation-add', 'nix3-derivation-show', 'nix3-develop', 'nix3-edit', + 'nix3-env', 'nix3-env-shell', 'nix3-eval', + 'nix3-flake', 'nix3-flake-archive', 'nix3-flake-check', 'nix3-flake-clone', 'nix3-flake-info', 'nix3-flake-init', 'nix3-flake-lock', - 'nix3-flake', 'nix3-flake-metadata', 'nix3-flake-new', 'nix3-flake-prefetch', + 'nix3-flake-prefetch-inputs', 'nix3-flake-show', 'nix3-flake-update', 'nix3-fmt', - 'nix3-hash-file', + 'nix3-formatter', + 'nix3-formatter-build', + 'nix3-formatter-run', 'nix3-hash', 'nix3-hash-convert', + 'nix3-hash-file', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -294,15 +292,21 @@ nix3_manpages = [ 'nix3-hash-to-sri', 'nix3-help', 'nix3-help-stores', + 'nix3-key', + 'nix3-key-convert-public-to-pem', + 'nix3-key-convert-secret-to-pem', 'nix3-key-convert-secret-to-public', 'nix3-key-generate-secret', - 'nix3-key', 'nix3-log', + 'nix3-nar', 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', 'nix3-nar-pack', - 'nix3-nar', + 'nix3-nario', + 'nix3-nario-export', + 'nix3-nario-import', + 'nix3-nario-list', 'nix3-path-info', 'nix3-print-dev-env', 'nix3-profile', @@ -314,19 +318,26 @@ nix3_manpages = [ 'nix3-profile-rollback', 'nix3-profile-upgrade', 'nix3-profile-wipe-history', - 'nix3-realisation-info', - 'nix3-realisation', + 'nix3-provenance', + 'nix3-provenance-show', + 'nix3-provenance-verify', + 'nix3-ps', + 'nix3-registry', 'nix3-registry-add', 'nix3-registry-list', - 'nix3-registry', 'nix3-registry-pin', 'nix3-registry-remove', + 'nix3-registry-resolve', 'nix3-repl', 'nix3-run', 'nix3-search', + 'nix3-serve', + 'nix3-store', 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', + 'nix3-store-build-trace-info', + 'nix3-store-build-trace', 'nix3-store-cat', 'nix3-store-copy-log', 'nix3-store-copy-sigs', @@ -337,16 +348,15 @@ nix3_manpages = [ 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', - 'nix3-store', 'nix3-store-optimise', 'nix3-store-path-from-hash-part', 'nix3-store-prefetch-file', 'nix3-store-repair', + 'nix3-store-roots-daemon', 'nix3-store-sign', 'nix3-store-verify', 'nix3-upgrade-nix', 'nix3-why-depends', - 'nix', ] foreach page : nix3_manpages diff --git a/doc/manual/package.nix b/doc/manual/package.nix index af5e6cf1c229..c66163e285e3 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -34,7 +34,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-manual"; + pname = "determinate-nix-manual"; inherit version; workDir = ./.; @@ -42,6 +42,7 @@ mkMesonDerivation (finalAttrs: { fileset.difference (fileset.unions [ ../../.version + ../../.version-determinate # For example JSON ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash @@ -144,6 +145,21 @@ mkMesonDerivation (finalAttrs: { # Exclude undocumented builtins ".*/language/builtins\\.html#builtins-addErrorContext" ".*/language/builtins\\.html#builtins-appendContext" + # `print.html` aggregates content from all pages, including + # the JSON schema pages and builtins pages excluded above, + # so it inherits the same broken fragment links. + ".*/print\\.html#algorithm" + ".*/print\\.html#root" + ".*/print\\.html#builtins-addErrorContext" + ".*/print\\.html#builtins-appendContext" + ".*/print\\.html#derivations_pattern1_structuredAttrs_additionalProperties" + ".*/print\\.html#structuredAttrs_additionalProperties" + ]; + # `404.html` uses `` so that absolute links + # work on the deployed site. Lychee cannot resolve `/` against + # a local file path, so skip the file entirely. + exclude_path = [ + ".*/404\\.html" ]; }; }; diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 0a6c71508006..76ce242178f3 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -243,29 +243,11 @@ "gloss-validity": "glossary.html#gloss-validity", "part-glossary": "glossary.html", "sec-building-source": "installation/building-source.html", - "ch-env-variables": "installation/env-variables.html", - "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables", - "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file", - "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file", "chap-installation": "installation/index.html", - "ch-installing-binary": "installation/installing-binary.html", - "sect-macos-installation": "installation/installing-binary.html#macos-installation", - "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation", - "sect-macos-installation-encrypted-volume": "installation/installing-binary.html#macos-installation", - "sect-macos-installation-recommended-notes": "installation/installing-binary.html#macos-installation", - "sect-macos-installation-symlink": "installation/installing-binary.html#macos-installation", - "sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation", - "sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball", - "sect-nix-install-pinned-version-url": - "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url", - "sect-single-user-installation": "installation/installing-binary.html#single-user-installation", "ch-installing-source": "installation/installing-source.html", - "ssec-multi-user": "installation/multi-user.html", "ch-nix-security": "installation/nix-security.html", "sec-obtaining-source": "installation/obtaining-source.html", "sec-prerequisites-source": "installation/prerequisites-source.html", - "sec-single-user": "installation/single-user.html", - "ch-supported-platforms": "installation/supported-platforms.html", "ch-upgrading-nix": "installation/upgrading.html", "ch-about-nix": "introduction.html", "chap-introduction": "introduction.html", @@ -287,43 +269,7 @@ "sec-sharing-packages": "package-management/sharing-packages.html", "ssec-ssh-substituter": "package-management/ssh-substituter.html", "chap-quick-start": "quick-start.html", - "sec-relnotes": "release-notes/index.html", - "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html", - "ch-relnotes-0.10": "release-notes/rl-0.10.html", - "ssec-relnotes-0.11": "release-notes/rl-0.11.html", - "ssec-relnotes-0.12": "release-notes/rl-0.12.html", - "ssec-relnotes-0.13": "release-notes/rl-0.13.html", - "ssec-relnotes-0.14": "release-notes/rl-0.14.html", - "ssec-relnotes-0.15": "release-notes/rl-0.15.html", - "ssec-relnotes-0.16": "release-notes/rl-0.16.html", - "ch-relnotes-0.5": "release-notes/rl-0.5.html", - "ch-relnotes-0.6": "release-notes/rl-0.6.html", - "ch-relnotes-0.7": "release-notes/rl-0.7.html", - "ch-relnotes-0.8.1": "release-notes/rl-0.8.1.html", - "ch-relnotes-0.8": "release-notes/rl-0.8.html", - "ch-relnotes-0.9.1": "release-notes/rl-0.9.1.html", - "ch-relnotes-0.9.2": "release-notes/rl-0.9.2.html", - "ch-relnotes-0.9": "release-notes/rl-0.9.html", - "ssec-relnotes-1.0": "release-notes/rl-1.0.html", - "ssec-relnotes-1.1": "release-notes/rl-1.1.html", - "ssec-relnotes-1.10": "release-notes/rl-1.10.html", - "ssec-relnotes-1.11.10": "release-notes/rl-1.11.10.html", - "ssec-relnotes-1.11": "release-notes/rl-1.11.html", - "ssec-relnotes-1.2": "release-notes/rl-1.2.html", - "ssec-relnotes-1.3": "release-notes/rl-1.3.html", - "ssec-relnotes-1.4": "release-notes/rl-1.4.html", - "ssec-relnotes-1.5.1": "release-notes/rl-1.5.html", - "ssec-relnotes-1.5.2": "release-notes/rl-1.5.2.html", - "ssec-relnotes-1.5": "release-notes/rl-1.5.html", - "ssec-relnotes-1.6.1": "release-notes/rl-1.6.1.html", - "ssec-relnotes-1.6.0": "release-notes/rl-1.6.html", - "ssec-relnotes-1.7": "release-notes/rl-1.7.html", - "ssec-relnotes-1.8": "release-notes/rl-1.8.html", - "ssec-relnotes-1.9": "release-notes/rl-1.9.html", - "ssec-relnotes-2.0": "release-notes/rl-2.0.html", - "ssec-relnotes-2.1": "release-notes/rl-2.1.html", - "ssec-relnotes-2.2": "release-notes/rl-2.2.html", - "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + "sec-relnotes": "release-notes/index.html" }, "language/types.html": { "simple-values": "#primitives", @@ -337,15 +283,13 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { - "linux": "uninstall.html#linux", - "macos": "uninstall.html#macos", "uninstalling": "uninstall.html" }, "development/building.html": { - "nix-with-flakes": "#building-nix-with-flakes", + "nix-with-flakes": "#building-nix", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", "unit-tests": "testing.html#unit-tests", @@ -353,7 +297,6 @@ "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", "integration-tests": "testing.html#integration-tests", "installer-tests": "testing.html#installer-tests", - "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", "characterization-testing": "testing.html#characterisation-testing-unit", @@ -363,6 +306,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0ce7..a0fcb6a55776 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/rl-next/shorter-build-dir-names.md b/doc/manual/rl-next/shorter-build-dir-names.md new file mode 100644 index 000000000000..e87fa5d04fb8 --- /dev/null +++ b/doc/manual/rl-next/shorter-build-dir-names.md @@ -0,0 +1,6 @@ +--- +synopsis: "Temporary build directories no longer include derivation names" +prs: [13839] +--- + +Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`. \ No newline at end of file diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 18d24788ca1c..5b16561a04aa 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -3,25 +3,21 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) - [Installation](installation/index.md) - - [Supported Platforms](installation/supported-platforms.md) - - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - - [Single-User Mode](installation/single-user.md) - - [Multi-User Mode](installation/multi-user.md) - - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -65,8 +61,11 @@ - [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - - [Main Commands](command-ref/main-commands.md) + - [Subcommands](command-ref/subcommands.md) +{{#include ./command-ref/new-cli/SUMMARY.md}} + - [Deprecated Commands](command-ref/main-commands.md) - [nix-build](command-ref/nix-build.md) + - [nix-channel](command-ref/nix-channel.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) @@ -102,22 +101,17 @@ - [nix-env --uninstall](command-ref/nix-env/uninstall.md) - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) - [nix-copy-closure](command-ref/nix-copy-closure.md) - [nix-daemon](command-ref/nix-daemon.md) - [nix-hash](command-ref/nix-hash.md) - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - - [Experimental Commands](command-ref/experimental-commands.md) -{{#include ./command-ref/new-cli/SUMMARY.md}} - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Profiles](command-ref/files/profiles.md) - [manifest.nix](command-ref/files/manifest.nix.md) - [manifest.json](command-ref/files/manifest.json.md) - - [Channels](command-ref/files/channels.md) - - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) @@ -125,8 +119,10 @@ - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) - [Store Path](protocols/json/store-path.md) + - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) + - [Resolved Derivation](protocols/json/derivation/resolved.md) - [Derivation Options](protocols/json/derivation/options.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) @@ -135,9 +131,13 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) + - [`builtins.wasm` Host Interface](protocols/wasm.md) + - [Flake Schemas](protocols/flake-schemas.md) - [C API](c-api.md) - [Glossary](glossary.md) - [Development](development/index.md) @@ -147,12 +147,78 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) + - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) - [Contributing](development/contributing.md) -- [Releases](release-notes/index.md) +- [Determinate Nix Release Notes](release-notes-determinate/index.md) + - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.21.9 (2026-07-30)](release-notes-determinate/v3.21.9.md) + - [Release 3.21.8 (2026-07-20)](release-notes-determinate/v3.21.8.md) + - [Release 3.21.7 (2026-07-15)](release-notes-determinate/v3.21.7.md) + - [Release 3.21.6 (2026-07-14)](release-notes-determinate/v3.21.6.md) + - [Release 3.21.5 (2026-07-08)](release-notes-determinate/v3.21.5.md) + - [Release 3.21.2 (2026-06-18)](release-notes-determinate/v3.21.2.md) + - [Release 3.21.1 (2026-06-08)](release-notes-determinate/v3.21.1.md) + - [Release 3.21.0 (2026-05-22)](release-notes-determinate/v3.21.0.md) + - [Release 3.20.0 (2026-05-07)](release-notes-determinate/v3.20.0.md) + - [Release 3.19.1 (2026-05-04)](release-notes-determinate/v3.19.1.md) + - [Release 3.19.0 (2026-04-28)](release-notes-determinate/v3.19.0.md) + - [Release 3.18.1 (2026-04-23)](release-notes-determinate/v3.18.1.md) + - [Release 3.18.0 (2026-04-20)](release-notes-determinate/v3.18.0.md) + - [Release 3.17.3 (2026-04-07)](release-notes-determinate/v3.17.3.md) + - [Release 3.17.2 (2026-03-27)](release-notes-determinate/v3.17.2.md) + - [Release 3.17.1 (2026-03-18)](release-notes-determinate/v3.17.1.md) + - [Release 3.17.0 (2026-03-04)](release-notes-determinate/v3.17.0.md) + - [Release 3.16.3 (2026-02-24)](release-notes-determinate/v3.16.3.md) + - [Release 3.16.2 (2026-02-23)](release-notes-determinate/v3.16.2.md) + - [Release 3.16.1 (2026-02-22)](release-notes-determinate/v3.16.1.md) + - [Release 3.16.0 (2026-02-12)](release-notes-determinate/v3.16.0.md) + - [Release 3.15.2 (2026-01-20)](release-notes-determinate/v3.15.2.md) + - [Release 3.15.1 (2025-12-24)](release-notes-determinate/v3.15.1.md) + - [Release 3.15.0 (2025-12-19)](release-notes-determinate/v3.15.0.md) + - [Release 3.14.0 (2025-12-08)](release-notes-determinate/v3.14.0.md) + - [Release 3.13.2 (2025-11-19)](release-notes-determinate/v3.13.2.md) + - [Release 3.13.1 (2025-11-12)](release-notes-determinate/v3.13.1.md) + - [Release 3.13.0 (2025-11-09)](release-notes-determinate/v3.13.0.md) + - [Release 3.12.2 (2025-11-05)](release-notes-determinate/v3.12.2.md) + - [Release 3.12.1 (2025-11-04)](release-notes-determinate/v3.12.1.md) + - [Release 3.12.0 (2025-10-23)](release-notes-determinate/v3.12.0.md) + - [Release 3.11.3 (2025-10-09)](release-notes-determinate/v3.11.3.md) + - [Release 3.11.2 (2025-09-12)](release-notes-determinate/v3.11.2.md) + - [Release 3.11.1 (2025-09-04)](release-notes-determinate/v3.11.1.md) + - [Release 3.11.0 (2025-09-03)](release-notes-determinate/v3.11.0.md) + - [Release 3.10.1 (2025-09-02)](release-notes-determinate/v3.10.1.md) + - [Release 3.10.0 (2025-09-02)](release-notes-determinate/v3.10.0.md) + - [Release 3.9.1 (2025-08-28)](release-notes-determinate/v3.9.1.md) + - [Release 3.9.0 (2025-08-26)](release-notes-determinate/v3.9.0.md) + - [Release 3.8.6 (2025-08-19)](release-notes-determinate/v3.8.6.md) + - [Release 3.8.5 (2025-08-04)](release-notes-determinate/rl-3.8.5.md) + - [Release 3.8.4 (2025-07-21)](release-notes-determinate/rl-3.8.4.md) + - [Release 3.8.3 (2025-07-18)](release-notes-determinate/rl-3.8.3.md) + - [Release 3.8.2 (2025-07-12)](release-notes-determinate/rl-3.8.2.md) + - [Release 3.8.1 (2025-07-11)](release-notes-determinate/rl-3.8.1.md) + - [Release 3.8.0 (2025-07-10)](release-notes-determinate/rl-3.8.0.md) + - [Release 3.7.0 (2025-07-03)](release-notes-determinate/rl-3.7.0.md) + - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md) + - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) + - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) + - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) + - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) + - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) + - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) + - [Release 3.5.2 (2025-05-12)](release-notes-determinate/rl-3.5.2.md) + - [Release 3.5.1 (2025-05-09)](release-notes-determinate/rl-3.5.1.md) + - [~~Release 3.5.0 (2025-05-09)~~](release-notes-determinate/rl-3.5.0.md) + - [Release 3.4.2 (2025-05-05)](release-notes-determinate/rl-3.4.2.md) + - [Release 3.4.0 (2025-04-25)](release-notes-determinate/rl-3.4.0.md) + - [Release 3.3.0 (2025-04-11)](release-notes-determinate/rl-3.3.0.md) + - [Release 3.1.0 (2025-03-27)](release-notes-determinate/rl-3.1.0.md) + - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) +- [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) @@ -162,60 +228,3 @@ - [Release 2.28 (2025-04-02)](release-notes/rl-2.28.md) - [Release 2.27 (2025-03-03)](release-notes/rl-2.27.md) - [Release 2.26 (2025-01-22)](release-notes/rl-2.26.md) - - [Release 2.25 (2024-11-07)](release-notes/rl-2.25.md) - - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) - - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) - - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md) - - [Release 2.0 (2018-02-22)](release-notes/rl-2.0.md) - - [Release 1.11.10 (2017-06-12)](release-notes/rl-1.11.10.md) - - [Release 1.11 (2016-01-19)](release-notes/rl-1.11.md) - - [Release 1.10 (2015-09-03)](release-notes/rl-1.10.md) - - [Release 1.9 (2015-06-12)](release-notes/rl-1.9.md) - - [Release 1.8 (2014-12-14)](release-notes/rl-1.8.md) - - [Release 1.7 (2014-04-11)](release-notes/rl-1.7.md) - - [Release 1.6.1 (2013-10-28)](release-notes/rl-1.6.1.md) - - [Release 1.6 (2013-09-10)](release-notes/rl-1.6.md) - - [Release 1.5.2 (2013-05-13)](release-notes/rl-1.5.2.md) - - [Release 1.5 (2013-02-27)](release-notes/rl-1.5.md) - - [Release 1.4 (2013-02-26)](release-notes/rl-1.4.md) - - [Release 1.3 (2013-01-04)](release-notes/rl-1.3.md) - - [Release 1.2 (2012-12-06)](release-notes/rl-1.2.md) - - [Release 1.1 (2012-07-18)](release-notes/rl-1.1.md) - - [Release 1.0 (2012-05-11)](release-notes/rl-1.0.md) - - [Release 0.16 (2010-08-17)](release-notes/rl-0.16.md) - - [Release 0.15 (2010-03-17)](release-notes/rl-0.15.md) - - [Release 0.14 (2010-02-04)](release-notes/rl-0.14.md) - - [Release 0.13 (2009-11-05)](release-notes/rl-0.13.md) - - [Release 0.12 (2008-11-20)](release-notes/rl-0.12.md) - - [Release 0.11 (2007-12-31)](release-notes/rl-0.11.md) - - [Release 0.10.1 (2006-10-11)](release-notes/rl-0.10.1.md) - - [Release 0.10 (2006-10-06)](release-notes/rl-0.10.md) - - [Release 0.9.2 (2005-09-21)](release-notes/rl-0.9.2.md) - - [Release 0.9.1 (2005-09-20)](release-notes/rl-0.9.1.md) - - [Release 0.9 (2005-09-16)](release-notes/rl-0.9.md) - - [Release 0.8.1 (2005-04-13)](release-notes/rl-0.8.1.md) - - [Release 0.8 (2005-04-11)](release-notes/rl-0.8.md) - - [Release 0.7 (2005-01-12)](release-notes/rl-0.7.md) - - [Release 0.6 (2004-11-14)](release-notes/rl-0.6.md) - - [Release 0.5 and earlier](release-notes/rl-0.5.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9ce9..74815df282c8 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! @@ -36,6 +37,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! @@ -46,6 +48,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/advanced-topics/distributed-builds.md b/doc/manual/source/advanced-topics/distributed-builds.md index 08a980643e88..c39cf4500795 100644 --- a/doc/manual/source/advanced-topics/distributed-builds.md +++ b/doc/manual/source/advanced-topics/distributed-builds.md @@ -5,8 +5,8 @@ this allows multiple builds to be performed in parallel. Remote builds also allow Nix to perform multi-platform builds in a semi-transparent way. For example, if you perform a build for a -`x86_64-darwin` on an `i686-linux` machine, Nix can automatically -forward the build to a `x86_64-darwin` machine, if one is available. +`aarch64-darwin` on an `x86_64-linux` machine, Nix can automatically +forward the build to a `aarch64-darwin` machine, if one is available. ## Requirements @@ -59,7 +59,7 @@ then you need to ensure that the `PATH` of non-interactive login shells contains Nix. The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. -For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: +For example, the following command allows you to build a derivation for `aarch64-darwin` on a Linux machine: ```console uname @@ -71,8 +71,8 @@ Linux ```console nix build --impure \ - --expr '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ - --builders 'ssh://mac x86_64-darwin' + --expr '(with import { system = "aarch64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac aarch64-darwin' ``` ```console @@ -90,12 +90,12 @@ Darwin It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. ```console - --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + --builders 'ssh://mac aarch64-darwin ; ssh://beastie x86_64-freebsd' ``` Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. - builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd + builders = ssh://mac aarch64-darwin ; ssh://beastie x86_64-freebsd After making changes to `nix.conf`, restart the Nix daemon for changes to take effect. @@ -107,4 +107,4 @@ file included in `builders` via the syntax `@/path/to/file`. For example, causes the list of machines in `/etc/nix/machines` to be included. (This is the default.) -[Nix instance]: @docroot@/glossary.md#gloss-nix-instance \ No newline at end of file +[Nix instance]: @docroot@/glossary.md#gloss-nix-instance diff --git a/doc/manual/source/command-ref/conf-file-prefix.md b/doc/manual/source/command-ref/conf-file-prefix.md index 627806cfbda8..53a3761fd4b1 100644 --- a/doc/manual/source/command-ref/conf-file-prefix.md +++ b/doc/manual/source/command-ref/conf-file-prefix.md @@ -8,19 +8,26 @@ Nix supports a variety of configuration settings, which are read from configurat ## Configuration file -By default Nix reads settings from the following places, in that order: +By default Nix reads settings from each of the following places. +Settings are applied on top of one another, so later settings overwrite earlier entries. -1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. +1. The system-wide configuration file `nix.conf` in the configuration directory. + + The configuration directory defaults to `${sysconfdir}/nix` (i.e. `/etc/nix` on most Unix systems) or `%PROGRAMDATA%\nix\conf` on Windows, and can be overridden with [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR). Values loaded in this file are not forwarded to the Nix daemon. The client assumes that the daemon has already loaded them. -1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. +1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path in the list will be loaded in reverse order. + The list separator is `:` on Unix and `;` on Windows. + + Otherwise it will look for `nix.conf` in: - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). - If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + - The [user configuration directory](./env-common.md#user-conf-dir) + - On Unix additionally, `nix/nix.conf` under each directory in [`XDG_CONFIG_DIRS`](./env-common.md#env-XDG_CONFIG_DIRS) 1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. + In particular, settings are separated by newlines, just as in the configuration file. ### File format diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 7be064a13804..cc7fe77eae56 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -1,34 +1,25 @@ # Common Environment Variables -Most Nix commands interpret the following environment variables: +Most Nix commands interpret the following environment variables. -- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) +## Configuration environment variables - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. +The following environment variables affect how Nix loads its configuration. +See the [configuration file](@docroot@/command-ref/conf-file.md#configuration-file) documentation for full details. -- [`NIX_PATH`](#env-NIX_PATH) +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) - A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Overrides the system configuration directory. - This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) - It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + Overrides the user configuration file locations. - > **Example** - > - > ```bash - > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos - > ``` +- [`NIX_CONFIG`](#env-NIX_CONFIG) - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + Provides configuration settings inline. - > **Example** - > - > ```bash - > $ NIX_PATH= nix-instantiate --eval '' - > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) - > ``` +## Store setting environment variables - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) @@ -55,37 +46,34 @@ Most Nix commands interpret the following environment variables: - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) - Overrides the location of the Nix store (default `prefix/store`). + Overrides the location of the Nix store. + On Unix, the default is `/nix/store`, set at compile time by the `libstore:store-dir` build option. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\store`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). -- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) - - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). + See the [Store Types] chapter; each store has a `store` setting linking back to this as its default. -- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - - Overrides the location of the Nix state directory (default - `prefix/var/nix`). +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) -- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) + Overrides the location of the Nix log directory. + On Unix, the default is `/nix/var/log/nix`, set at compile time by the `log-dir` build option. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\log`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). - Overrides the location of the system Nix configuration directory - (default `sysconfdir/nix`, i.e. `/etc/nix` on most systems). + The [Local Store], [Local Daemon Store], and [Experimental SSH Store with filesystem mounted] have per-store settings that override this. -- [`NIX_CONFIG`](#env-NIX_CONFIG) +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. + Overrides the location of the Nix state directory. + On Unix, the default is `${localstatedir}/nix`, where `localstatedir` is a compile-time build option defaulting to `/nix/var`. + On Windows, there is no compile-time setting; the default is `%PROGRAMDATA%\nix\state`, determined at runtime using the [`%PROGRAMDATA%` Known Folder](#known-folders). -- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + The [Local Store], [Local Daemon Store], and [Experimental SSH Store with filesystem mounted] have per-store settings that override this. - Overrides the location of the Nix user configuration files to load from. +- [`NIX_DAEMON_SOCKET_PATH`](#env-NIX_DAEMON_SOCKET_PATH) - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + Overrides the path to the Unix domain socket used to communicate with the Nix daemon. + Defaults to `daemon-socket/socket` within the state directory (see [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)). - The variable is treated as a list separated by the `:` token. + See the [Local Daemon Store] documentation for details on how the socket path is resolved. - [`TMPDIR`](#env-TMPDIR) @@ -95,12 +83,15 @@ Most Nix commands interpret the following environment variables: - [`NIX_REMOTE`](#env-NIX_REMOTE) - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. + Overrides the [`store`](@docroot@/command-ref/conf-file.md#conf-store) setting. + +[Store Types]: @docroot@/store/types/index.md +[store URL format]: @docroot@/store/types/index.md#store-url-format +[Local Store]: @docroot@/store/types/local-store.md +[Local Daemon Store]: @docroot@/store/types/local-daemon-store.md +[Experimental SSH Store with filesystem mounted]: @docroot@/store/types/experimental-ssh-store-with-filesystem-mounted.md + +## Nix language evaluator settings - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) @@ -120,32 +111,87 @@ Most Nix commands interpret the following environment variables: 384 MiB. Setting it to a low value reduces memory consumption, but will increase runtime due to the overhead of garbage collection. -## XDG Base Directories +- [`NIX_PATH`](#env-NIX_PATH) -Nix follows the [XDG Base Directory Specification]. + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). -For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled. -[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default. + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). -The following environment variables are used to determine locations of various state and configuration files: + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). -- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) -- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) -- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) + > **Example** + > + > ```bash + > $ export NIX_PATH=/home/eelco/Dev:nixos-config=/etc/nixos + > ``` -[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html -[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` + +## User Directories + +Nix supports per-user **configuration**, **state**, and **cache** directories. +The following environment variables override the locations for these directories: + +- [user configuration directory]{#user-conf-dir}: [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} +- user state directory: [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} +- user cache directory: [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} -In addition, setting the following environment variables overrides the XDG base directories: +When these are not set, the defaults depend on the platform: -- [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} (default `$XDG_CONFIG_HOME/nix`) -- [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} (default `$XDG_STATE_HOME/nix`) -- [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} (default `$XDG_CACHE_HOME/nix`) +- On Unix, the [XDG base directories](#xdg-base-directories): `$XDG_CONFIG_HOME/nix`, `$XDG_STATE_HOME/nix`, `$XDG_CACHE_HOME/nix` +- On Windows, the [Windows Known Folders](#known-folders): `%APPDATA%\nix\config`, `%LOCALAPPDATA%\nix\state`, `%LOCALAPPDATA%\nix\cache` -When [`use-xdg-base-directories`] is enabled, the configuration directory is: +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories + +For backwards compatibility, legacy Nix commands (e.g. `nix-env`, `nix-channel`) use dotfiles in `$HOME` instead of these directories unless [`use-xdg-base-directories`] is enabled. +[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) use the proper directories by default. + +When [`use-xdg-base-directories`] is enabled, the configuration directory is resolved as: 1. `$NIX_CONFIG_HOME`, if it is defined -2. Otherwise, `$XDG_CONFIG_HOME/nix`, if `XDG_CONFIG_HOME` is defined -3. Otherwise, `~/.config/nix`. +2. Otherwise, the platform default (e.g. `$XDG_CONFIG_HOME/nix` on Unix) Likewise for the state and cache directories. + +## Miscellaneous environment variables + +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) + + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. + +## Appendix: OS-specific conventions + +This information is not Nix-specific, but is referenced above. + +### Unix: XDG Base Directories {#xdg-base-directories} + +The [XDG Base Directory Specification] defines standard locations for user-specific configuration, state, and cache files on Unix systems. + +[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +The following environment variables are used: + +- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) +- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) +- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) +- [`XDG_CONFIG_DIRS`]{#env-XDG_CONFIG_DIRS} (default `/etc/xdg`) — colon-separated list of additional configuration base directories, searched after `XDG_CONFIG_HOME` + +### Windows: Known Folders {#known-folders} + +On Windows, [Known Folders][windows-known-folders] provide standard locations for application data on Windows. + +[windows-known-folders]: https://learn.microsoft.com/en-us/windows/win32/shell/known-folders + +The relevant folders are: + +- [`%APPDATA%`]{#env-APPDATA} — per-user roaming application data +- [`%LOCALAPPDATA%`]{#env-LOCALAPPDATA} — per-user local application data +- [`%PROGRAMDATA%`]{#env-PROGRAMDATA} — system-wide application data diff --git a/doc/manual/source/command-ref/experimental-commands.md b/doc/manual/source/command-ref/experimental-commands.md deleted file mode 100644 index 1190729a2305..000000000000 --- a/doc/manual/source/command-ref/experimental-commands.md +++ /dev/null @@ -1,8 +0,0 @@ -# Experimental Commands - -This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). - -> **Warning** -> -> These commands may be removed in the future, or their syntax may -> change in incompatible ways. diff --git a/doc/manual/source/command-ref/files/default-nix-expression.md b/doc/manual/source/command-ref/files/default-nix-expression.md index cd73b8e58ab7..66ce84b48f8b 100644 --- a/doc/manual/source/command-ref/files/default-nix-expression.md +++ b/doc/manual/source/command-ref/files/default-nix-expression.md @@ -31,12 +31,12 @@ Then, the resulting expression is interpreted like this: The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. -The command [`nix-channel`] places a symlink to the current user's [channels] in this directory, the [user channel link](#user-channel-link). +The command [`nix-channel`] places a symlink to the current user's channels in this directory, the [user channel link](#user-channel-link). This makes all subscribed channels available as attributes in the default expression. ## User channel link -A symlink that ensures that [`nix-env`] can find the current user's [channels]: +A symlink that ensures that [`nix-env`] can find the current user's channels: - `~/.nix-defexpr/channels` - `$XDG_STATE_HOME/nix/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. @@ -51,4 +51,3 @@ In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, [`nix-channel`]: @docroot@/command-ref/nix-channel.md [`nix-env`]: @docroot@/command-ref/nix-env.md [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories -[channels]: @docroot@/command-ref/files/channels.md diff --git a/doc/manual/source/command-ref/files/profiles.md b/doc/manual/source/command-ref/files/profiles.md index 4f022acceb3e..f137336747f1 100644 --- a/doc/manual/source/command-ref/files/profiles.md +++ b/doc/manual/source/command-ref/files/profiles.md @@ -67,7 +67,7 @@ By default, this symlink points to: - `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root` The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user. -The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`]. +The installer sets this up by default, unless you enable [`use-xdg-base-directories`]. [`nix-env`]: @docroot@/command-ref/nix-env.md [`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index 865f43ccce5b..59817be974b5 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -8,6 +8,12 @@ # Description +> **Warning** +> +> nix-channel is deprecated in favor of flakes in Determinate Nix. +> For a guide on Nix flakes, see: . +> For details and to offer feedback on the deprecation process, see: . + Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. The moving parts of channels are: diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee18..07229255e7cd 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the diff --git a/doc/manual/source/command-ref/nix-env.md b/doc/manual/source/command-ref/nix-env.md index bda02149ed06..d01caaf7f787 100644 --- a/doc/manual/source/command-ref/nix-env.md +++ b/doc/manual/source/command-ref/nix-env.md @@ -52,7 +52,7 @@ These pages can be viewed offline: `nix-env` can obtain packages from multiple sources: - An attribute set of derivations from: - - The [default Nix expression](@docroot@/command-ref/files/default-nix-expression.md) (by default) + - The default Nix expression (by default) - A Nix file, specified via `--file` - A [profile](@docroot@/command-ref/files/profiles.md), specified via `--from-profile` - A Nix expression that is a function which takes default expression as argument, specified via `--from-expression` diff --git a/doc/manual/source/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md index 06aeb3925628..320fa530fdaa 100644 --- a/doc/manual/source/command-ref/nix-env/install.md +++ b/doc/manual/source/command-ref/nix-env/install.md @@ -22,12 +22,11 @@ It is based on the current generation of the active [profile](@docroot@/command- The arguments *args* map to store paths in a number of possible ways: -- By default, *args* is a set of names denoting derivations in the [default Nix expression]. +- By default, *args* is a set of names denoting derivations in the default Nix expression. These are [realised], and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. [derivation expression]: @docroot@/glossary.md#gloss-derivation-expression - [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that @@ -45,7 +44,7 @@ The arguments *args* map to store paths in a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). -- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the default Nix expression. This is faster than using derivation names and unambiguous. Show the attribute paths of available packages with [`nix-env --query`](./query.md): @@ -58,7 +57,7 @@ The arguments *args* map to store paths in a number of possible ways: easy way to copy user environment elements from one profile to another. -- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the default Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095b4..c1a4251b0973 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad46d..86c20b9e1de4 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md index aeb696535989..cc45eeb74cf1 100644 --- a/doc/manual/source/command-ref/nix-store/query.md +++ b/doc/manual/source/command-ref/nix-store/query.md @@ -103,6 +103,13 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. + > **Note** + > + > `nix-store --query --deriver` is replaced with the following `nix` command: + > + > nix path-info --json ... | jq -r '.[].deriver' + + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers` diff --git a/doc/manual/source/command-ref/subcommands.md b/doc/manual/source/command-ref/subcommands.md new file mode 100644 index 000000000000..6a26732338d1 --- /dev/null +++ b/doc/manual/source/command-ref/subcommands.md @@ -0,0 +1,3 @@ +# Subcommands + +This section lists all the subcommands of the `nix` CLI. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 742170f76c6c..917e39e1ca5f 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -1,77 +1,9 @@ # Building Nix -This section provides some notes on how to start hacking on Nix. -To get the latest version of Nix from GitHub: - > **Note** > > When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo. -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -> **Note** -> -> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. -> If you don't have it installed, follow the [installation instructions](../installation/index.md). - - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenv -``` - -> **Note** -> -> You can use `native-ccacheStdenv` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ out="$(pwd)/outputs/out" dev=$out debug=$out mesonFlags+=" --prefix=${out}" -[nix-shell]$ dontAddPrefix=1 configurePhase -[nix-shell]$ buildPhase -``` - -To test it: - -```console -[nix-shell]$ checkPhase -``` - -To install it in `$(pwd)/outputs`: - -```console -[nix-shell]$ installPhase -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - -## Building Nix with flakes - -This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. - -[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-feature-nix-command - To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console @@ -130,8 +62,6 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: [`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix - `x86_64-linux` -- `x86_64-darwin` -- `i686-linux` - `aarch64-linux` - `aarch64-darwin` - `armv6l-linux` @@ -149,12 +79,6 @@ platform. Common solutions include [remote build machines] and [binary format em Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: -```console -$ nix-build --attr packages.aarch64-linux.default -``` - -or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: - ```console $ nix build .#packages.aarch64-linux.default ``` @@ -223,6 +147,7 @@ For historic reasons and backward-compatibility, some CPU and OS identifiers are |-----------------------------|-------------------------|---------------------| | `x86` | | `i686` | | `arm` | | `host_machine.cpu()`| +| `arm64` | | `host_machine.cpu()`| | `ppc` | `little` | `powerpcle` | | `ppc64` | `little` | `powerpc64le` | | `ppc` | `big` | `powerpc` | @@ -247,20 +172,12 @@ To build with one of those environments, you can use $ nix build .#nix-cli-ccacheStdenv ``` -for flake-enabled Nix, or - -```console -$ nix-build --attr nix-cli-ccacheStdenv -``` - -for classic Nix. - You can use any of the other supported environments in place of `nix-cli-ccacheStdenv`. ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#building-nix-with-flakes) or in [classic Nix](#building-nix). +See [supported compilation environments](#compilation-environments) and instructions how to [set up a shell](#building-nix). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06e4..a9d131147588 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d991a..35e4c71ec388 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/doc/manual/source/development/experimental-features.md b/doc/manual/source/development/experimental-features.md index ad5cffa91ee5..56a45b23890a 100644 --- a/doc/manual/source/development/experimental-features.md +++ b/doc/manual/source/development/experimental-features.md @@ -6,7 +6,7 @@ Experimental features are considered unstable, which means that they can be chan Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). This allows accessing unstable functionality without unwittingly relying on it. -Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Experimental feature flags were first introduced in [Nix 2.4](https://nix.dev/manual/nix/latest/release-notes/rl-2.4). Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. This was a source of confusion and controversy. diff --git a/doc/manual/source/development/static-analysis.md b/doc/manual/source/development/static-analysis.md new file mode 100644 index 000000000000..6bdf875befde --- /dev/null +++ b/doc/manual/source/development/static-analysis.md @@ -0,0 +1,87 @@ +# Static Analysis + +Nix uses [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) for static analysis of C++ code. +This helps catch bugs, enforce coding standards, and maintain code quality. + +## Running clang-tidy locally + +To run clang-tidy on the entire codebase in the development shell: + +```console +$ nix develop .#native-clangStdenv +$ configurePhase +$ meson compile -C build clang-tidy +``` + +This will analyze all C++ source files and report any warnings. + +To automatically apply fixes for certain warnings: + +```console +$ meson compile -C build clang-tidy-fix +``` + +> **Warning** +> +> Review the changes before committing, as automatic fixes may not always be correct. + +## CI integration + +clang-tidy runs automatically on every pull request via GitHub Actions. +The CI job builds `.#hydraJobs.clangTidy.x86_64-linux` which: + +1. Builds all components with debug mode (for faster compilation) +2. Runs clang-tidy on each component +3. Fails if any warnings are found (warnings are treated as errors) + +## Configuration + +The clang-tidy configuration is in `.clang-tidy` at the repository root (symlinked from `nix-meson-build-support/common/clang-tidy/.clang-tidy`). + +### Suppressing warnings + +If a warning is a false positive, you can suppress it in several ways: + +1. **Inline suppression** (preferred for specific cases): + ```cpp + // NOLINTBEGIN(bugprone-some-check) + ... code ... + // NOLINTEND(bugprone-some-check) + ``` + Or for a single line: + ```cpp + int x = something(); // NOLINT(bugprone-some-check) + ``` + +2. **Configuration file** (for project-wide suppression): + Add the check to the disabled list in `.clang-tidy`: + ```yaml + Checks: + - -bugprone-some-check # Reason for disabling + ``` + +3. **Check options** (for configuring check behavior): + ```yaml + CheckOptions: + bugprone-reserved-identifier.AllowedIdentifiers: '__some_identifier' + ``` + +### Adding new checks + +To enable additional checks: + +1. Edit `nix-meson-build-support/common/clang-tidy/.clang-tidy` +2. Add the check to the `Checks` list +3. Run clang-tidy locally to see the impact +4. Fix any new warnings or disable specific sub-checks if needed + +## Custom clang-tidy plugin + +The Nix project includes infrastructure for custom clang-tidy checks in `src/clang-tidy-plugin/`. +These checks can enforce Nix-specific coding patterns that aren't covered by standard clang-tidy checks. + +To add a new custom check: + +1. Add the check implementation in `src/clang-tidy-plugin/` +2. Register it in `nix-clang-tidy-checks.cc` +3. Enable it in `.clang-tidy` with the `nix-` prefix diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index dd965862a34c..6ea1a233a4aa 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -9,7 +9,7 @@ You can build it yourself: ``` # nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html +# xdg-open ./result/index.html ``` [Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. @@ -34,31 +34,28 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > │ ├── value/context.cc > │ … > │ -> ├── tests -> │ │ +> ├── libutil-tests +> │ ├── meson.build > │ … -> │ ├── libutil-tests -> │ │ ├── meson.build -> │ │ … -> │ │ └── data -> │ │ ├── git/tree.txt -> │ │ … -> │ │ -> │ ├── libexpr-test-support +> │ ├── data +> │ │ ├── git/tree.txt +> │ … +> │ +> ├── libexpr-test-support +> │ ├── meson.build +> │ ├── include/nix/expr > │ │ ├── meson.build -> │ │ ├── include/nix/expr -> │ │ │ ├── meson.build -> │ │ │ └── tests -> │ │ │ ├── value/context.hh -> │ │ │ … > │ │ └── tests -> │ │ ├── value/context.cc +> │ │ ├── value/context.hh > │ │ … -> │ │ -> │ ├── libexpr-tests -> │ … ├── meson.build +> │ ├── tests > │ ├── value/context.cc > │ … +> │ +> ├── libexpr-tests +> │ ├── meson.build +> │ ├── value/context.cc +> │ … > … > ``` @@ -80,7 +77,7 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. -A environment variables that Google Test accepts are also worth knowing: +The environment variables that Google Test accepts are also worth knowing: 1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) @@ -257,15 +254,6 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. -### Troubleshooting - -Sometimes running tests in the development shell may leave artefacts in the local repository. -To remove any traces of that: - -```console -git clean -x --force tests -``` - ### Characterisation testing { #characterisation-testing-functional } Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. @@ -311,78 +299,34 @@ Generally, this build is sufficient, but in nightly or CI we also test the attri The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). +Because these tests are expensive and require more than what the standard github-actions setup provides, most of them only run on the master branch (on ). You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. ## Installer tests -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +GitHub Actions CI in the Nix repository also tests the installer on PRs. It does not require additional setup and utilises [GHA Artifacts](https://docs.github.com/en/actions/tutorials/store-and-share-data) and can be run in any Nix repository fork. -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: +- The `tests` job generates installers for the platforms below and uploads them as an artifact: - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. + - `aarch64-darwin` -### One-time setup +- The `installer_test` job (which runs on Linux and macOS) will try to install Nix with the cached installer and run a trivial Nix command. +- Both the scripted installer and the [standalone Rust-based installer](https://github.com/NixOS/nix-installer) are tested. -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. +You can generate the installer tarball and script manually by running `nix build .#hydraJobs.installerScriptForGHA.`. ## Working on documentation ### Using the CI-generated installer for manual testing -After the CI run completes, you can check the output to extract the installer URL: +After the CI run completes, you can check the output to extract the installer artifact: 1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: +2. Scroll down to `Artifacts` section. +3. Download the corresponding installer artifact (`installer-darwin` for `aarch64-darwin` and `installer-linux` for `x86_64-linux`). +4. Unpack the downloaded `.zip` artifact. +5. To generate an install command, plug the path to the unpacked artifact into this template: ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + sh /install --tarball-url-prefix file:// ``` - - - diff --git a/doc/manual/source/favicon.png b/doc/manual/source/favicon.png deleted file mode 100644 index 1ed2b5fe0fdf..000000000000 Binary files a/doc/manual/source/favicon.png and /dev/null differ diff --git a/doc/manual/source/favicon.svg b/doc/manual/source/favicon.svg index 1d2a6e835d5f..55fb9479b06e 100644 --- a/doc/manual/source/favicon.svg +++ b/doc/manual/source/favicon.svg @@ -1 +1,29 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index af5b45314d1d..c7b8931509a8 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -1,5 +1,32 @@ # Glossary +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory + +- [binary cache]{#gloss-binary-cache} + + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + - [build system]{#gloss-build-system} Generic term for software that facilitates the building of software by automating the invocation of compilers, linkers, and other tools. @@ -8,6 +35,26 @@ It has no knowledge of any particular programming language or toolchain. These details are specified in [derivation expressions](#gloss-derivation-expression). +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its [output path] is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + See [References](@docroot@/store/store-object.md#references) for details. + + [closure]: #gloss-closure + - [content address]{#gloss-content-address} A @@ -31,32 +78,37 @@ The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata. +- [content-addressed store object]{#gloss-content-addressed-store-object} + + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + +- [content-addressing derivation]{#gloss-content-addressing-derivation} + + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + - [derivation]{#gloss-derivation} A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. - There are multiple ways of obtaining store objects from store derivatons, collectively called [realisation][realise]. + There are multiple ways of obtaining store objects from store derivations, collectively called [realisation][realise]. [derivation]: #gloss-derivation -- [store derivation]{#gloss-store-derivation} - - A [derivation] represented as a [store object]. - - See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - - [store derivation]: #gloss-store-derivation - -- [directed acyclic graph]{#gloss-directed-acyclic-graph} +- [derivation expression]{#gloss-derivation-expression} - A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. + A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - DAGs are very important to Nix. - In particular, the non-self-[references][reference] of [store object][store object] form a cycle. + [derivation expression]: #gloss-derivation-expression - [derivation path]{#gloss-derivation-path} @@ -68,79 +120,34 @@ [derivation path]: #gloss-derivation-path -- [derivation expression]{#gloss-derivation-expression} - - A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - - [derivation expression]: #gloss-derivation-expression - -- [instantiate]{#gloss-instantiate}, instantiation - - Translate a [derivation expression] into a [store derivation]. - - See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. - - [instantiate]: #gloss-instantiate - -- [realise]{#gloss-realise}, realisation - - Ensure a [store path] is [valid][validity]. - - This can be achieved by: - - Fetching a pre-built [store object] from a [substituter] - - [Building](@docroot@/store/building.md) the corresponding [store derivation] - - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs - - See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - - See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - - [realise]: #gloss-realise - -- [content-addressing derivation]{#gloss-content-addressing-derivation} - - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. - -- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - - A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [deriver]{#gloss-deriver} -- [store]{#gloss-store} + The [store derivation] that produced an [output path]. - A collection of [store objects][store object], with operations to manipulate that collection. - See [Nix Store](./store/index.md) for details. + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). - There are many types of stores, see [Store Types](./store/types/index.md) for details. +- [deriving path]{#gloss-deriving-path} - [store]: #gloss-store + Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. -- [Nix instance]{#gloss-nix-instance} - - 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. - A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. - 2. A running Nix process, such as the `nix` command. + See [Deriving Path](./store/derivation/index.md#deriving-path) for details. -- [binary cache]{#gloss-binary-cache} + Not to be confused with [derivation path]. - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [directed acyclic graph]{#gloss-directed-acyclic-graph} -- [store path]{#gloss-store-path} + A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. - The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + DAGs are very important to Nix. + In particular, the non-self-[references][reference] of [store object][store object] form a cycle. - > **Example** - > - > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` +- [experimental feature]{#gloss-experimental-feature} - See [Store Path](@docroot@/store/store-path.md) for details. + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - [store path]: #gloss-store-path + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). - [file system object]{#gloss-file-system-object} @@ -150,21 +157,19 @@ [file system object]: #gloss-file-system-object -- [store object]{#gloss-store-object} - - Part of the contents of a [store]. - - A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. - It can be referred to by a [store path]. - - See [Store Object](@docroot@/store/store-object.md) for details. +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - [store object]: #gloss-store-object + A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a @@ -174,42 +179,28 @@ See [input-addressing derivation outputs](store/derivation/outputs/input-address.md) for details. -- [content-addressed store object]{#gloss-content-addressed-store-object} - - A [store object] which is [content-addressed](#gloss-content-address), - i.e. whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - - See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. - -- [substitute]{#gloss-substitute} +- [installable]{#gloss-installable} - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Something that can be realised in the Nix store. -- [substituter]{#gloss-substituter} + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. +- [instantiate]{#gloss-instantiate}, instantiation - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + Translate a [derivation expression] into a [store derivation]. - [substituter]: #gloss-substituter + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. -- [purity]{#gloss-purity} + [instantiate]: #gloss-instantiate - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. +- [Nix Archive (NAR)]{#gloss-nar} -- [impure derivation]{#gloss-impure-derivation} + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. - [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, - so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. - [Nix database]{#gloss-nix-database} @@ -235,145 +226,136 @@ > > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. -- [reference]{#gloss-reference} - - An edge from one [store object] to another. +- [Nix instance]{#gloss-nix-instance} + + 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. + A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. + 2. A running Nix process, such as the `nix` command. - See [References](@docroot@/store/store-object.md#references) for details. +- [output]{#gloss-output} - [reference]: #gloss-reference + A [store object] produced by a [store derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. - See [References](@docroot@/store/store-object.md#references) for details. + [output]: #gloss-output -- [reachable]{#gloss-reachable} +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. +- [output path]{#gloss-output-path} - See [References](@docroot@/store/store-object.md#references) for details. + The [store path] to the [output] of a [store derivation]. -- [closure]{#gloss-closure} + [output path]: #gloss-output-path - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its [output path] is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. +- [package]{#package} - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + A software package; files that belong together for a particular purpose, and metadata. - See [References](@docroot@/store/store-object.md#references) for details. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - [closure]: #gloss-closure + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), + - attributes with metadata, such as information about how the package is supposed to be used. -- [requisite]{#gloss-requisite} + The exact shape of these attribute sets is up to convention. - A store object [reachable] by a path (chain of references) from a given [store object]. - The [closure] is the set of requisites. + [package]: #package - See [References](@docroot@/store/store-object.md#references) for details. +- [profile]{#gloss-profile} -- [referrer]{#gloss-referrer} + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. - A reversed edge from one [store object] to another. +- [purity]{#gloss-purity} -- [output]{#gloss-output} + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - A [store object] produced by a [store derivation]. - See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. +- [reachable]{#gloss-reachable} - [output]: #gloss-output + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. -- [output path]{#gloss-output-path} + See [References](@docroot@/store/store-object.md#references) for details. - The [store path] to the [output] of a [store derivation]. +- [realise]{#gloss-realise}, realisation - [output path]: #gloss-output-path + Ensure a [store path] is [valid][validity]. -- [output closure]{#gloss-output-closure}\ - The [closure] of an [output path]. It only contains what is [reachable] from the output. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - [Building](@docroot@/store/building.md) the corresponding [store derivation] + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs -- [deriving path]{#gloss-deriving-path} + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - See [Deriving Path](./store/derivation/index.md#deriving-path) for details. + [realise]: #gloss-realise - Not to be confused with [derivation path]. +- [reference]{#gloss-reference} -- [deriver]{#gloss-deriver} + An edge from one [store object] to another. - The [store derivation] that produced an [output path]. + See [References](@docroot@/store/store-object.md#references) for details. - The deriver for an output path can be queried with the `--deriver` option to - [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + [reference]: #gloss-reference -- [validity]{#gloss-validity} + See [References](@docroot@/store/store-object.md#references) for details. - A store path is valid if all [store object]s in its [closure] can be read from the [store]. +- [referrer]{#gloss-referrer} - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. + A reversed edge from one [store object] to another. - [validity]: #gloss-validity - [local store]: @docroot@/store/types/local-store.md +- [requisite]{#gloss-requisite} -- [user environment]{#gloss-user-env} + A store object [reachable] by a path (chain of references) from a given [store object]. + The [closure] is the set of requisites. - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. + See [References](@docroot@/store/store-object.md#references) for details. -- [profile]{#gloss-profile} +- [store]{#gloss-store} - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. -- [installable]{#gloss-installable} + There are many types of stores, see [Store Types](./store/types/index.md) for details. - Something that can be realised in the Nix store. + [store]: #gloss-store - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. +- [store derivation]{#gloss-store-derivation} -- [Nix Archive (NAR)]{#gloss-nar} + A [derivation] represented as a [store object]. - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + [store derivation]: #gloss-store-derivation -- [`∅`]{#gloss-empty-set} +- [store object]{#gloss-store-object} - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + Part of the contents of a [store]. -- [`ε`]{#gloss-epsilon} + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + See [Store Object](@docroot@/store/store-object.md) for details. -- [package]{#package} + [store object]: #gloss-store-object - A software package; files that belong together for a particular purpose, and metadata. +- [store path]{#gloss-store-path} - Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - - attributes with metadata, such as information about how the package is supposed to be used. + > **Example** + > + > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` - The exact shape of these attribute sets is up to convention. + See [Store Path](@docroot@/store/store-path.md) for details. - [package]: #package + [store path]: #gloss-store-path - [string interpolation]{#gloss-string-interpolation} @@ -385,31 +367,47 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs -- [base directory]{#gloss-base-directory} +- [SRI]{#gloss-sri} - The location from which relative paths are resolved. + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. - - For expressions in a file, the base directory is the directory containing that file. - This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). - + [SRI]: #gloss-sri - - - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. +- [substitute]{#gloss-substitute} - [base directory]: #gloss-base-directory + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. -- [experimental feature]{#gloss-experimental-feature} +- [substituter]{#gloss-substituter} - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + + [substituter]: #gloss-substituter +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [validity]{#gloss-validity} + + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md [Nix language]: ./language/index.md diff --git a/doc/manual/source/installation/env-variables.md b/doc/manual/source/installation/env-variables.md deleted file mode 100644 index 0350904211ac..000000000000 --- a/doc/manual/source/installation/env-variables.md +++ /dev/null @@ -1,62 +0,0 @@ -# Environment Variables - -To use Nix, some environment variables should be set. In particular, -`PATH` should contain the directories `prefix/bin` and -`~/.nix-profile/bin`. The first directory contains the Nix tools -themselves, while `~/.nix-profile` is a symbolic link to the current -*user environment* (an automatically generated package consisting of -symlinks to installed packages). The simplest way to set the required -environment variables is to include the file -`prefix/etc/profile.d/nix.sh` in your `~/.profile` (or similar), like -this: - -```bash -source prefix/etc/profile.d/nix.sh -``` - -# `NIX_SSL_CERT_FILE` - -If you need to specify a custom certificate bundle to account for an -HTTPS-intercepting man in the middle proxy, you must specify the path to -the certificate bundle in the environment variable `NIX_SSL_CERT_FILE`. - -If you don't specify a `NIX_SSL_CERT_FILE` manually, Nix will install -and use its own certificate bundle. - -Set the environment variable and install Nix - -```console -$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ curl -L https://nixos.org/nix/install | sh -``` - -In the shell profile and rc files (for example, `/etc/bashrc`, -`/etc/zshrc`), add the following line: - -```bash -export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -``` - -> **Note** -> -> You must not add the export and then do the install, as the Nix -> installer will detect the presence of Nix configuration, and abort. - -If you use the Nix daemon, you should also add the following to -`/etc/nix/nix.conf`: - -``` -ssl-cert-file = /etc/ssl/my-certificate-bundle.crt -``` - -## Proxy Environment Variables - -The Nix installer has special handling for these proxy-related -environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, -`ALL_PROXY`, `NO_PROXY`. - -If any of these variables are set when running the Nix installer, then -the installer will create an override file at -`/etc/systemd/system/nix-daemon.service.d/override.conf` so `nix-daemon` -will use them. diff --git a/doc/manual/source/installation/index.md b/doc/manual/source/installation/index.md index 3c09f103184a..aded684b0b59 100644 --- a/doc/manual/source/installation/index.md +++ b/doc/manual/source/installation/index.md @@ -1,44 +1,11 @@ # Installation -This section describes how to install and configure Nix for first-time use. - -The current recommended option on Linux and MacOS is [multi-user](#multi-user). - -## Multi-user - -This installation offers better sharing, improved isolation, and more security -over a single user installation. - -This option requires either: - -* Linux running systemd, with SELinux disabled -* MacOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon -``` - -## Single-user - -> Single-user is not supported on Mac. - -> `warning: installing Nix as root is not supported by this script!` - -This installation has less requirements than the multi-user install, however it -cannot offer equivalent sharing, isolation, or security. - -This option is suitable for systems without systemd. - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install ``` ## Distributions @@ -46,3 +13,5 @@ $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon The Nix community maintains installers for several distributions. They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/installation/installing-binary.md b/doc/manual/source/installation/installing-binary.md deleted file mode 100644 index 21c15637437d..000000000000 --- a/doc/manual/source/installation/installing-binary.md +++ /dev/null @@ -1,158 +0,0 @@ -# Installing a Binary Distribution - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -To install the latest version Nix, run the following command: - -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -This performs the default type of installation for your platform: - -- [Multi-user](#multi-user-installation): - - Linux with systemd and without SELinux - - macOS -- [Single-user](#single-user-installation): - - Linux without systemd - - Linux with SELinux - -We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. - -The installer can be configured with various command line arguments and environment variables. -To show available command line flags: - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --help -``` - -To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). - -# Installing a pinned Nix version from a URL - -Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). -The directory for each version contains the corresponding SHA-256 hash. - -All installation scripts are invoked the same way: - -```console -$ export VERSION=2.19.2 -$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh -``` - -# Multi User Installation - -The multi-user Nix installation creates system users and a system service for the Nix daemon. - -Supported systems: - -- Linux running systemd, with SELinux disabled -- macOS - -To explicitly instruct the installer to perform a multi-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --daemon -``` - -You can run this under your usual user account or `root`. -The script will invoke `sudo` as needed. - -# Single User Installation - -To explicitly select a single-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --no-daemon -``` - -In a single-user installation, `/nix` is owned by the invoking user. -The script will invoke `sudo` to create `/nix` if it doesn’t already exist. -If you don’t have `sudo`, manually create `/nix` as `root`: - -```console -$ su root -# mkdir /nix -# chown alice /nix -``` - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) -- Download and unpack the tarball -- Run the installer - -> **Example** -> -> ```console -> $ pushd $(mktemp -d) -> $ export VERSION=2.19.2 -> $ export SYSTEM=x86_64-linux -> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz -> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz -> $ cd nix-$VERSION-$SYSTEM -> $ ./install -> $ popd -> ``` - -The installer can be customised with the environment variables declared in the file named `install-multi-user`. - -## Native packages for Linux distributions - -The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). - -# macOS Installation - - -[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root file system -on modern macOS. New installs will do this automatically. - -This section previously detailed the situation, options, and trade-offs, -but it now only outlines what the installer does. You don't need to know -this to run the installer, but it may help if you run into trouble: - -- create a new APFS volume for your Nix store -- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" - empty root directory to mount your volume -- specify mount options for the volume in `/etc/fstab` - - `rw`: read-write - - `noauto`: prevent the system from auto-mounting the volume (so the - LaunchDaemon mentioned below can control mounting it, and to avoid - masking problems with that mounting service). - - `nobrowse`: prevent the Nix Store volume from showing up on your - desktop; also keeps Spotlight from spending resources to index - this volume - -- if you have FileVault enabled - - generate an encryption password - - put it in your system Keychain - - use it to encrypt the volume -- create a system LaunchDaemon to mount this volume early enough in the - boot process to avoid problems loading or restoring any programs that - need access to your Nix store - diff --git a/doc/manual/source/installation/nix-security.md b/doc/manual/source/installation/nix-security.md index 1e9036b68b21..61cad24c2b3b 100644 --- a/doc/manual/source/installation/nix-security.md +++ b/doc/manual/source/installation/nix-security.md @@ -1,15 +1,85 @@ # Security -Nix has two basic security models. First, it can be used in “single-user -mode”, which is similar to what most other package management tools do: -there is a single user (typically root) who performs all package -management operations. All other users can then use the installed -packages, but they cannot perform package management operations -themselves. - -Alternatively, you can configure Nix in “multi-user mode”. In this -model, all users can perform package management operations — for -instance, every user can install software without requiring root -privileges. Nix ensures that this is secure. For instance, it’s not -possible for one user to overwrite a package used by another user with a -Trojan horse. +Nix follows a [**multi-user**](#multi-user-model) security model in which all +users can perform package management operations. Every user can, for example, +install software without requiring root privileges, and Nix ensures that this +is secure. It's *not* possible for one user to, for example, overwrite a +package used by another user with a Trojan horse. + +## Multi-User model + +To allow a Nix store to be shared safely among multiple users, it is +important that users are not able to run builders that modify the Nix +store or database in arbitrary ways, or that interfere with builds +started by other users. If they could do so, they could install a Trojan +horse in some package and compromise the accounts of other users. + +To prevent this, the Nix store and database are owned by some privileged +user (usually `root`) and builders are executed under special user +accounts (usually named `nixbld1`, `nixbld2`, etc.). When a unprivileged +user runs a Nix command, actions that operate on the Nix store (such as +builds) are forwarded to a *Nix daemon* running under the owner of the +Nix store/database that performs the operation. + +> **Note** +> +> Multi-user mode has one important limitation: only root and a set of +> trusted users specified in `nix.conf` can specify arbitrary binary +> caches. So while unprivileged users may install packages from +> arbitrary Nix expressions, they may not get pre-built binaries. + +### Setting up the build users + +The *build users* are the special UIDs under which builds are performed. +They should all be members of the *build users group* `nixbld`. This +group should have no other members. The build users should not be +members of any other group. On Linux, you can create the group and users +as follows: + +```console +$ groupadd -r nixbld +$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \ + -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \ + nixbld$n; done +``` + +This creates 10 build users. There can never be more concurrent builds +than the number of build users, so you may want to increase this if you +expect to do many builds at the same time. + +### Running the daemon + +The [Nix daemon](../command-ref/nix-daemon.md) should be started as +follows (as `root`): + +```console +$ nix-daemon +``` + +You’ll want to put that line somewhere in your system’s boot scripts. + +To let unprivileged users use the daemon, they should set the +[`NIX_REMOTE` environment variable](../command-ref/env-common.md) to +`daemon`. So you should put a line like + +```console +export NIX_REMOTE=daemon +``` + +into the users’ login scripts. + +### Restricting access + +To limit which users can perform Nix operations, you can use the +permissions on the directory `/nix/var/nix/daemon-socket`. For instance, +if you want to restrict the use of Nix to the members of a group called +`nix-users`, do + +```console +$ chgrp nix-users /nix/var/nix/daemon-socket +$ chmod ug=rwx,o= /nix/var/nix/daemon-socket +``` + +This way, users who are not in the `nix-users` group cannot connect to +the Unix domain socket `/nix/var/nix/daemon-socket/socket`, so they +cannot perform Nix operations. diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 057fd444349e..e98067348ee6 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. diff --git a/doc/manual/source/installation/single-user.md b/doc/manual/source/installation/single-user.md deleted file mode 100644 index f9a3b26edf41..000000000000 --- a/doc/manual/source/installation/single-user.md +++ /dev/null @@ -1,9 +0,0 @@ -# Single-User Mode - -In single-user mode, all Nix operations that access the database in -`prefix/var/nix/db` or modify the Nix store in `prefix/store` must be -performed under the user ID that owns those directories. This is -typically root. (If you install from RPM packages, that’s in fact the -default ownership.) However, on single-user machines, it is often -convenient to `chown` those directories to your normal user account so -that you don’t have to `su` to root all the time. diff --git a/doc/manual/source/installation/supported-platforms.md b/doc/manual/source/installation/supported-platforms.md deleted file mode 100644 index 8ca3ce8d445e..000000000000 --- a/doc/manual/source/installation/supported-platforms.md +++ /dev/null @@ -1,7 +0,0 @@ -# Supported Platforms - -Nix is currently supported on the following platforms: - - - Linux (i686, x86\_64, aarch64). - - - macOS (x86\_64, aarch64). diff --git a/doc/manual/source/installation/uninstall.md b/doc/manual/source/installation/uninstall.md index 4bb1c0e6ef48..e95634c213a1 100644 --- a/doc/manual/source/installation/uninstall.md +++ b/doc/manual/source/installation/uninstall.md @@ -1,196 +1,15 @@ # Uninstalling Nix -## Multi User +To uninstall Determinate Nix, use the uninstallation utility built into the [Determinate Nix Installer][installer]: -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. - -### Linux - -If you are on Linux with systemd: - -1. Remove the Nix daemon service: - - ```console - sudo systemctl stop nix-daemon.service - sudo systemctl disable nix-daemon.socket nix-daemon.service - sudo systemctl daemon-reload - ``` - -2. Remove files created by Nix: - - ```console - sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~/.local/share/nix ~/.local/state/nix ~/.cache/nix ~/.nix-defexpr ~/.nix-profile ~/.nix-channels ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix - ``` - -3. Remove build users and their group: - - ```console - for i in $(seq 1 32); do - sudo userdel nixbld$i - done - sudo groupdel nixbld - ``` - -4. There may also be references to Nix in - - `/etc/bash.bashrc` - - `/etc/bashrc` - - `/etc/profile` - - `/etc/zsh/zshrc` - - `/etc/zshrc` - - which you may remove. - -### FreeBSD - -1. Stop and remove the Nix daemon service: - - ```console - sudo service nix-daemon stop - sudo rm -f /usr/local/etc/rc.d/nix-daemon - sudo sysrc -x nix_daemon_enable - ``` - -2. Remove files created by Nix: - - ```console - sudo rm -rf /etc/nix /usr/local/etc/profile.d/nix.sh /nix ~/.local/share/nix ~/.local/state/nix ~/.cache/nix ~/.nix-defexpr ~/.nix-profile ~/.nix-channels ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix - ``` - -3. Remove build users and their group: - - ```console - for i in $(seq 1 32); do - sudo pw userdel nixbld$i - done - sudo pw groupdel nixbld - ``` - -4. There may also be references to Nix in: - - `/usr/local/etc/bashrc` - - `/usr/local/etc/zshrc` - - Shell configuration files in users' home directories - - which you may remove. - -### macOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like - - ``` - UUID= /nix apfs rw,noauto,nobrowse,suid,owners - ``` - or - - ``` - LABEL=Nix\040Store /nix apfs rw,nobrowse - ``` - - by setting the cursor on the respective line using the arrow keys, and pressing `dd`, and then `:wq` to save the file. - - This will prevent automatic mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. - If this is the only line in the file you can remove it entirely: - - ```bash - if [ -f /etc/synthetic.conf ]; then - if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then - sudo rm /etc/synthetic.conf - else - sudo vi /etc/synthetic.conf - fi - fi - ``` - - This will prevent the creation of the empty `/nix` directory. - -6. Remove the files Nix added to your system, except for the store: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels ~/.local/share/nix ~/.local/state/nix ~/.cache/nix - ``` - - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the store. - - If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. - Look for a "Nix Store" volume in the output of the following command: - - ```console - diskutil list - ``` - - If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier. - - If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` directory. -> This is an expected sign of a successful uninstall. -> The empty `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. -> The uninstall is complete. -> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. - -## Single User +```console +$ /nix/nix-installer uninstall +``` -To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: +If you're certain that you want to uninstall, you can skip the confirmation step: ```console -rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile ~/.local/share/nix ~/.local/state/nix ~/.cache/nix +$ /nix/nix-installer uninstall --no-confirm ``` -You might also want to manually remove references to Nix from your `~/.profile`. + +[installer]: https://github.com/DeterminateSystems/nix-installer diff --git a/doc/manual/source/installation/upgrading.md b/doc/manual/source/installation/upgrading.md index a433f1d30e6c..8fe342b09b7c 100644 --- a/doc/manual/source/installation/upgrading.md +++ b/doc/manual/source/installation/upgrading.md @@ -1,40 +1,10 @@ # Upgrading Nix -> **Note** -> -> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md). - -Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`: - -```console -$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version" -nix (Nix) 2.18.1 -``` - -> **Warning** -> -> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! -> Reverting to an older version of Nix may therefore require purging the store database before it can be used. - -## Linux multi-user +You can upgrade Determinate Nix using Determinate Nixd: ```console -$ sudo su -# nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -# systemctl daemon-reload -# systemctl restart nix-daemon +sudo determinate-nixd upgrade ``` -## macOS multi-user +Note that the `sudo` is necessary here and upgrading fails without it. -```console -$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -$ sudo launchctl remove org.nixos.nix-daemon -$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist -``` - -## Single-user all platforms - -```console -$ nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -``` diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c917..06518598aa10 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -1,4 +1,19 @@ -# Introduction +# Determinate Nix + +**Determinate Nix** is a downstream distribution of [Nix], a purely functional language, CLI tool, and package management system. +It's available on Linux, macOS, and Windows Subsystem for Linux (WSL). + +## Installing + +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: + +```console +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install +``` + +## How Nix works Nix is a _purely functional package manager_. This means that it treats packages like values in a purely functional programming language @@ -10,7 +25,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -49,7 +64,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support @@ -174,7 +189,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS @@ -184,10 +199,14 @@ to build configuration files in `/etc`). This means, among other things, that it is easy to roll back the entire configuration of the system to an earlier state. Also, users can install software without root privileges. For more information and downloads, see the [NixOS -homepage](https://nixos.org/). +homepage][nix]. ## License Nix is released under the terms of the [GNU LGPLv2.1 or (at your option) any later -version](http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html). +version][license]. + +[license]: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal +[site]: https://nixos.org diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8a4..cc2743dc5ca9 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2d2..50aa525acbf4 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d6d..bc2cdfb27ab2 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8def..53cf5061f834 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 000000000000..d86d307ad9ee --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 000000000000..e2e2efac0eeb --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1cebe8..3859b2a10ac4 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663e3..778614eb1602 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/flake-schemas.md b/doc/manual/source/protocols/flake-schemas.md new file mode 100644 index 000000000000..f6a89f7a5d32 --- /dev/null +++ b/doc/manual/source/protocols/flake-schemas.md @@ -0,0 +1,87 @@ +# Flake Schemas + +Flake schemas are a mechanism to allow tools like `nix flake show` and `nix flake check` to enumerate and check the contents of a flake +in a generic way, without requiring built-in knowledge of specific flake output types like `packages` or `nixosConfigurations`. + +A flake can define schemas for its outputs by defining a `schemas` output. `schemas` should be an attribute set with an attribute for +every output type that you want to be supported. If a flake does not have a `schemas` attribute, Nix uses a built-in set of schemas (namely https://github.com/DeterminateSystems/flake-schemas). + +A schema is an attribute set with the following attributes: + +| Attribute | Description | Default | +| :---------------- | :----------------------------------------------------------------------------------------------------------------------------| :------ | +| `version` | Should be set to 1 | | +| `doc` | A string containing documentation about the flake output type in Markdown format. | | +| `allowIFD` | Whether the evaluation of the output attributes of this flake can read from derivation outputs. | `true` | +| `inventory` | A function that returns the contents of the flake output (described [below](#inventory)). | | +| `roles` | The roles supported by this flake output type (see [below](#roles)). | | +| `appendSystem` | Whether the current system type is appended to the flake output attribute path, as in outputs like `packages`. | | +| `defaultAttrPath` | A default flake output attribute path suffix. For example, `packages` will look for `default` if no attribute path is given. | | + +# Inventory + +The `inventory` function returns a _node_ describing the contents of the flake output. A node is either a _leaf node_ or a _non-leaf node_. This allows nested flake output attributes to be described (e.g. `x86_64-linux.hello` inside a `packages` output). + +Non-leaf nodes must have the following attribute: + +| Attribute | Description | +| :--------- | :------------------------------------------------------------------------------------- | +| `children` | An attribute set of nodes. If this attribute is missing, the attribute is a leaf node. | + +Leaf nodes can have the following attributes: + +| Attribute | Description | +| :------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `derivationAttrPath` | If not null, a list of strings denoting the attribute path of the "main" derivation of this node. | +| `evalChecks` | An attribute set of Boolean values, used by `nix flake check`. Each attribute must evaluate to `true`. | +| `isFlakeCheck` | Whether `nix flake check` should build the attribute denoted by `derivationAttrPath`. | +| `shortDescription` | A one-sentence description of the node (such as the `meta.description` attribute in Nixpkgs). | +| `what` | A brief human-readable string describing the type of the node, e.g. `"package"` or `"development environment"`. This is used by tools like `nix flake show` to describe the contents of a flake. | + +Both leaf and non-leaf nodes can have the following attributes: + +| Attribute | Description | +| :----------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `forSystems` | A list of Nix system types (e.g. `["x86_64-linux"]`) supported by this node. This is used by tools to skip nodes that cannot be built on the user's system. Setting this on a non-leaf node allows all the children to be skipped, regardless of the `forSystems` attributes of the children. If this attribute is not set, the node is never skipped. | +| `isLegacy` | If set to true, this node is skipped unless the `--legacy` CLI flag is set. | + +# Roles + +Roles allow schemas to declare what commands operate on them. For instance, to have the `nix build` command build a flake output, the schema should declare: +```nix +roles.nix-build = { }; +``` + +The following roles are used by various Nix commands: + +* `nix-build`: Used by `nix build`, `nix shell`, and `nix develop`. +* `nix-bundler`: Denotes bundler functions used by `nix bundle`. +* `nix-develop`: Used by `nix develop`. +* `nix-fmt`: Used by `nix formatter`. +* `nix-run`: Used by `nix run` and `nix bundle`. +* `nix-search`: Denotes an output that will be searched by `nix search`. +* `nix-template`: Used by `nix flake init` and `nix flake new`. + +Tools are free to define new roles. For instance, instead of hard-coding a flake output type like `nixosConfigurations`, `nixos-rebuild --flake` could use any flake output that implements a `nixos-configuration` role. + +# Example + +Here is a schema that checks that every element of the `nixosConfigurations` flake output evaluates and builds correctly (meaning that it has a `config.system.build.toplevel` attribute that yields a buildable derivation). + +```nix +outputs = { + schemas.nixosConfigurations = { + version = 1; + doc = '' + The `nixosConfigurations` flake output defines NixOS system configurations. + ''; + inventory = output: { + children = builtins.mapAttrs (configName: machine: + { + what = "NixOS configuration"; + derivationAttrPath = [ "config" "system" "build" "toplevel" ]; + }) output; + }; + }; +}; +``` diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 9eea93712601..d4a3a96edb64 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -1,21 +1,21 @@ -{{#include build-trace-entry-v2-fixed.md}} +{{#include build-trace-entry-v3-fixed.md}} ## Examples ### Simple build trace entry ```json -{{#include schema/build-trace-entry-v2/simple.json}} +{{#include schema/build-trace-entry-v3/simple.json}} ``` ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v2/with-signature.json}} +{{#include schema/build-trace-entry-v3/with-signature.json}} ``` diff --git a/doc/manual/source/protocols/json/derivation/resolved.md b/doc/manual/source/protocols/json/derivation/resolved.md new file mode 100644 index 000000000000..45e18d9af047 --- /dev/null +++ b/doc/manual/source/protocols/json/derivation/resolved.md @@ -0,0 +1 @@ +{{#include ../derivation-resolved-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 96b6f1801a57..0162036d420b 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -16,3 +16,6 @@ s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](@docroot@/proto s^\(./hash-v1.yaml\)^[JSON format for `Hash`](@docroot@/protocols/json/hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html#method)^g s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html)^g +s^\(./store-path-v1.yaml\)^[JSON format for `StorePath`](@docroot@/protocols/json/store-path.html)^g +s^\(./derivation-v4.yaml\)\?#/\$defs/common/properties/[a-zA-Z]*^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g +s^\(./derivation-v4.yaml\)^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c7c48c4ed6e2..9fcacb20387f 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,11 +13,13 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v2', + 'signature-v2', + 'store-object-info-v3', 'derivation-v4', + 'derivation-resolved-v4', 'derivation-options-v1', 'deriving-path-v1', - 'build-trace-entry-v2', + 'build-trace-entry-v3', 'build-result-v1', 'store-v1', ] diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 55e55d3e5cbe..d0d8d8a0c742 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml" + "$ref": "build-trace-entry-v3.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml deleted file mode 100644 index 4340f82388c2..000000000000 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ /dev/null @@ -1,95 +0,0 @@ -"$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json" -title: Build Trace Entry -description: | - A record of a successful build outcome for a specific derivation output. - - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). - - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) - > and subject to change. - - Verision history: - - - Version 1: Original format - - - Version 2: Remove `dependentRealisations` - -type: object -required: - - id - - outPath - - signatures -allOf: - - "$ref": "#/$defs/key" - - "$ref": "#/$defs/value" -properties: - id: {} - outPath: {} - signatures: {} -additionalProperties: - dependentRealisations: - description: deprecated field - type: object - -"$defs": - key: - title: Build Trace Key - description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. - type: object - required: - - id - properties: - id: - type: string - title: Derivation Output ID - pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" - description: | - Unique identifier for the derivation output that was built. - - Format: `{hash-quotient-drv}!{output-name}` - - - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). - Begins with `sha256:`. - - - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") - - Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` - - value: - title: Build Trace Value - description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "value" part, describing an output. - type: object - required: - - outPath - - signatures - properties: - outPath: - "$ref": "store-path-v1.yaml" - title: Output Store Path - description: | - The path to the store object that resulted from building this derivation for the given output name. - - patternProperties: - "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": - "$ref": "store-path-v1.yaml" - title: Dependent Store Path - description: Store path that this dependency resolved to during the build - additionalProperties: false - - signatures: - type: array - title: Build Signatures - description: | - A set of cryptographic signatures attesting to the authenticity of this build trace entry. - items: - type: string - title: Signature - description: A single cryptographic signature diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2 b/doc/manual/source/protocols/json/schema/build-trace-entry-v3 similarity index 100% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2 rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3 diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml new file mode 100644 index 000000000000..2e825af6fb8a --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -0,0 +1,83 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v3.json" +title: Build Trace Entry +description: | + A record of a successful build outcome for a specific derivation output. + + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + + ## Version History + + - Version 1: Original format + + - Version 2: + - Remove `dependentRealisations` + + - Version 3: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. + - Separate into `key` and `value` + - Use 2nd version of signatures format (objects, not strings) + +type: object +required: + - key + - value +properties: + key: + "$ref": "#/$defs/key" + value: + "$ref": "#/$defs/value" +additionalProperties: false + +"$defs": + key: + title: Build Trace Key + description: | + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. + This is the "key" part, referring to a derivation and output. + type: object + required: + - drvPath + - outputName + properties: + drvPath: + "$ref": "store-path-v1.yaml" + title: Derivation Path + description: | + The store path of the derivation that was built. + outputName: + type: string + title: Output Name + description: | + Name of the specific output (e.g., "out", "dev", "doc") + additionalProperties: false + + value: + title: Build Trace Value + description: | + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. + This is the "value" part, describing an output. + type: object + required: + - outPath + - signatures + properties: + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + "$ref": "signature-v2.yaml" diff --git a/doc/manual/source/protocols/json/schema/derivation-options-v1.yaml b/doc/manual/source/protocols/json/schema/derivation-options-v1.yaml index 58ff070882ff..d247802cd6c2 100644 --- a/doc/manual/source/protocols/json/schema/derivation-options-v1.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-options-v1.yaml @@ -9,7 +9,7 @@ description: | > **Warning** > > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md) > and subject to change. type: object diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml new file mode 100644 index 000000000000..705156eb8451 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -0,0 +1,52 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-resolved-v4.json" +title: Resolved Derivation +description: | + Experimental JSON representation of a resolved Nix derivation (version 4). + + This schema describes the JSON representation of Nix's `BasicDerivation` type, + which is a derivation with all input derivation dependencies resolved to store paths. + This is the result of `Derivation::tryResolve`. + + This is called "version 4" because we wish to keep it in sync with the + [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), + but actually it is the first version for this format. + +type: object +required: + - name + - version + - outputs + - inputs + - system + - builder + - args + - env +properties: + name: { "$ref": "derivation-v4.yaml#/$defs/common/properties/name" } + version: { "$ref": "derivation-v4.yaml#/$defs/common/properties/version" } + outputs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/outputs" } + system: { "$ref": "derivation-v4.yaml#/$defs/common/properties/system" } + builder: { "$ref": "derivation-v4.yaml#/$defs/common/properties/builder" } + args: { "$ref": "derivation-v4.yaml#/$defs/common/properties/args" } + env: { "$ref": "derivation-v4.yaml#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/structuredAttrs" } + + inputs: + type: array + title: Input source paths + description: | + List of store paths on which this resolved derivation depends. + Since all derivation inputs have been resolved, only source paths remain. + + > **Example** + > + > ```json + > "inputs": [ + > "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh", + > "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" +additionalProperties: false diff --git a/doc/manual/source/protocols/json/schema/derivation-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml index c41eef31bfc6..fa443d982c25 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -9,7 +9,7 @@ description: | > **Warning** > > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md) > and subject to change. type: object @@ -23,57 +23,14 @@ required: - args - env properties: - name: - type: string - title: Derivation name - description: | - The name of the derivation. - Used when calculating store paths for the derivation’s outputs. - - version: - const: 4 - title: Format version (must be 4) - description: | - Must be `4`. - This is a guard that allows us to continue evolving this format. - The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - - Version 0: ATerm format - - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - - - Version 2: Separate `method` and `hashAlgo` fields in output specs - - - Version 3: Drop store dir from store paths, just include base name. - - - Version 4: Two cleanups, batched together to lesson churn: - - - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) - - - Use canonical content address JSON format for floating content addressed derivation outputs. - - Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. - - outputs: - type: object - title: Output specifications - description: | - Information about the output paths of the derivation. - This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. - - > **Example** - > - > ```json - > "outputs": { - > "out": { - > "method": "nar", - > "hashAlgo": "sha256", - > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - > } - > } - > ``` - additionalProperties: - "$ref": "#/$defs/output/overall" + name: { "$ref": "#/$defs/common/properties/name" } + version: { "$ref": "#/$defs/common/properties/version" } + outputs: { "$ref": "#/$defs/common/properties/outputs" } + system: { "$ref": "#/$defs/common/properties/system" } + builder: { "$ref": "#/$defs/common/properties/builder" } + args: { "$ref": "#/$defs/common/properties/args" } + env: { "$ref": "#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "#/$defs/common/properties/structuredAttrs" } inputs: type: object @@ -126,47 +83,105 @@ properties: - "$ref": "#/$defs/dynamicOutputs" additionalProperties: false additionalProperties: false +additionalProperties: false - system: - type: string - title: Build system type +"$defs": + common: + title: Common Derivation Fields description: | - The system type on which this derivation is to be built - (e.g. `x86_64-linux`). + Fields shared between `Derivation` and `BasicDerivation`. + properties: + name: + type: string + title: Derivation name + description: | + The name of the derivation. + Used when calculating store paths for the derivation's outputs. - builder: - type: string - title: Build program path - description: | - Absolute path of the program used to perform the build. - Typically this is the `bash` shell - (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + version: + const: 4 + title: Format version (must be 4) + description: | + Must be `4`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: - args: - type: array - title: Builder arguments - description: | - Command-line arguments passed to the `builder`. - items: - type: string + - Version 0: ATerm format - env: - type: object - title: Environment variables - description: | - Environment variables passed to the `builder`. - additionalProperties: - type: string + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - structuredAttrs: - title: Structured attributes - description: | - [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. - Structured attributes are JSON, and thus embedded as-is. - type: object - additionalProperties: true + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + title: Output specifications + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output/overall" + + system: + type: string + title: Build system type + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + title: Build program path + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + + args: + type: array + title: Builder arguments + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + title: Environment variables + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + title: Structured attributes + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true -"$defs": output: overall: title: Derivation Output diff --git a/doc/manual/source/protocols/json/schema/nar-info-v3 b/doc/manual/source/protocols/json/schema/nar-info-v3 new file mode 120000 index 000000000000..378215b4156b --- /dev/null +++ b/doc/manual/source/protocols/json/schema/nar-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/nar-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml new file mode 100644 index 000000000000..d94798902f47 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -0,0 +1,27 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/signature-v2.json" +title: Signature +description: | + A cryptographic signature along with the name of the key that produced it. + + This schema describes the JSON representation of signatures as used in various Nix JSON APIs. + + ## Version History + + - Version 1: Colon-separated string in the format `:` + + - Version 2: Structured object with `keyName` and `sig` fields + +type: object +required: + - keyName + - sig +properties: + keyName: + type: string + title: Key Name + description: The name of the key used to produce this signature + sig: + type: string + title: Signature Data + description: The raw signature bytes, Base64-encoded diff --git a/doc/manual/source/protocols/json/schema/signature-v2/simple.json b/doc/manual/source/protocols/json/schema/signature-v2/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2 b/doc/manual/source/protocols/json/schema/store-object-info-v2 deleted file mode 120000 index 36ca7f13db94..000000000000 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2 +++ /dev/null @@ -1 +0,0 @@ -../../../../../../src/libstore-tests/data/path-info/json-2 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3 b/doc/manual/source/protocols/json/schema/store-object-info-v3 new file mode 120000 index 000000000000..d5dae48fcf69 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/path-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/store-object-info-v2.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index 3ed7e99e28d8..bf4e4dafe144 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -1,17 +1,11 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" -title: Store Object Info v2 +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v3.json" +title: Store Object Info v3 description: | Information about a [store object](@docroot@/store/store-object.md). This schema describes the JSON representation of store object metadata as returned by commands like [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md). - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - ### Field Categories Store object information can come in a few different variations. @@ -50,10 +44,10 @@ $defs: properties: version: type: integer - const: 2 - title: Format version (must be 2) + const: 3 + title: Format version (must be 3) description: | - Must be `2`. + Must be `3`. This is a guard that allows us to continue evolving this format. Here is the rough version history: @@ -63,6 +57,8 @@ $defs: - Version 2: Use structured JSON type for `ca` + - Version 3: Use structured JSON type for `signatures` + path: "$ref": "./store-path-v1.yaml" title: Store Path @@ -107,7 +103,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: @@ -174,7 +170,7 @@ $defs: > This is an "impure" field that may not be included in certain contexts. items: - type: string + "$ref": "./signature-v2.yaml" # Computed closure fields closureSize: @@ -185,6 +181,15 @@ $defs: The total size of this store object and every other object in its [closure](@docroot@/glossary.md#gloss-closure). > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure. + + provenance: + oneOf: + - type: "null" + - type: object # FIXME + title: Provenance + description: | + An arbitrary JSON object containing provenance information about the store object, or `null` if not available. + additionalProperties: false narInfo: @@ -268,4 +273,13 @@ $defs: > This is an impure "`.narinfo`" field that may not be included in certain contexts. > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure. + + provenance: + oneOf: + - type: "null" + - type: object # FIXME + title: Provenance + description: | + An arbitrary JSON object containing provenance information about the store object, or `null` if not available. + additionalProperties: false diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index f1f58c2bf1ac..31760c64a617 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -6,12 +6,6 @@ description: | This schema describes the JSON representation of store paths as used in various Nix JSON APIs. - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - ## Format Store paths in JSON are represented as strings containing just the hash and name portion, without the store directory prefix. @@ -24,7 +18,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml index ebe61d9cb227..eaf5c624b0be 100644 --- a/doc/manual/source/protocols/json/schema/store-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -10,7 +10,7 @@ description: | > **Warning** > > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md) > and subject to change. type: object @@ -37,7 +37,7 @@ properties: - contents properties: info: - "$ref": "./store-object-info-v2.yaml#/$defs/impure" + "$ref": "./store-object-info-v3.yaml#/$defs/impure" title: Store Object Info description: | Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc. @@ -70,7 +70,7 @@ properties: "^[A-Za-z0-9+/]{43}=$": type: object additionalProperties: - "$ref": "./build-trace-entry-v2.yaml#/$defs/value" + "$ref": "./build-trace-entry-v3.yaml#/$defs/value" additionalProperties: false "$defs": diff --git a/doc/manual/source/protocols/json/signature.md b/doc/manual/source/protocols/json/signature.md new file mode 100644 index 000000000000..363b4ef11e0a --- /dev/null +++ b/doc/manual/source/protocols/json/signature.md @@ -0,0 +1,9 @@ +{{#include signature-v2-fixed.md}} + +## Examples + +### Simple signature + +```json +{{#include schema/signature-v2/simple.json}} +``` diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4ad83de00b3f..ea1cc3a9d223 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,45 +1,45 @@ -{{#include store-object-info-v2-fixed.md}} +{{#include store-object-info-v3-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v2/pure.json}} +{{#include schema/store-object-info-v3/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v2/impure.json}} +{{#include schema/store-object-info-v3/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v2/empty_pure.json}} +{{#include schema/store-object-info-v3/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v2/empty_impure.json}} +{{#include schema/store-object-info-v3/empty_impure.json}} ``` ### NAR info (minimal) ```json -{{#include schema/nar-info-v2/pure.json}} +{{#include schema/nar-info-v3/pure.json}} ``` ### NAR info (with binary cache fields) ```json -{{#include schema/nar-info-v2/impure.json}} +{{#include schema/nar-info-v3/impure.json}} ``` diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea24..d8da1e9952cf 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1c8..bdce19d1d62c 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/protocols/wasm.md b/doc/manual/source/protocols/wasm.md new file mode 100644 index 000000000000..ca67491cadd5 --- /dev/null +++ b/doc/manual/source/protocols/wasm.md @@ -0,0 +1,379 @@ +# Wasm Host Interface + +Nix provides a builtin for calling WebAssembly modules: `builtins.wasm`. This allows extending Nix with custom functionality written in languages that compile to WebAssembly (such as Rust). + +## Overview + +WebAssembly modules can interact with Nix values through a host interface that provides functions for creating and inspecting Nix values. The WASM module receives Nix values as opaque `ValueId` handles and uses host functions to work with them. + +The `builtins.wasm` builtin takes two arguments: +1. A configuration attribute set with the following attributes: + - `path` - Path to the WebAssembly module (required) + - `function` - Name of the Wasm function to call (required for non-WASI modules, not allowed for WASI modules) +2. The argument value to pass to the function + +WASI mode is automatically detected by checking if the module imports from `wasi_snapshot_preview1`. There are two calling conventions: + +- **Non-WASI mode** (no WASI imports) calls the Wasm export specified by `function` directly. The function receives its input as a `ValueId` parameter and returns a `ValueId`. +- **WASI mode** (when the module imports from `wasi_snapshot_preview1`) runs the WASI module's `_start` entry point. The input `ValueId` is passed as a command-line argument (`argv[1]`), and the result is returned by calling the `return_to_nix` host function. + +## Value IDs + +Nix values are represented in Wasm code as a `u32` referred to below as a `ValueId`. These are opaque handles that reference values managed by the Nix evaluator. Value ID 0 is reserved to represent a missing attribute lookup result. + +## Entry Points + +### Non-WASI Mode + +Non-WASI mode is used when the module does **not** import from `wasi_snapshot_preview1`. + +Usage: +```nix +builtins.wasm { + path = ; + function = ; +} +``` + +Every Wasm module used in non-WASI mode must export: +- A `memory` object that the host can use to read/write data. +- `nix_wasm_init_v1()`, a function that is called once when the module is instantiated. +- The entry point function, whose name is specified by the `function` attribute. It takes a single `ValueId` and returns a single `ValueId` (i.e. it has type `fn(arg: u32) -> u32`). + +### WASI Mode + +WASI mode is automatically used when the module imports a `wasi_snapshot_preview1` function. + +Usage: +```nix +builtins.wasm { + path = ; +} +``` + +Every WASI module must export: +- A `memory` object that the host can use to read/write data. +- `_start()`, the standard WASI entry point. This function takes no parameters. + +The input value is passed as a command-line argument: `argv[1]` is set to the decimal representation of the `ValueId` of the input value. + +To return a result to Nix, the module must call the `return_to_nix` host function (see below) with the `ValueId` of the result. If `_start` finishes without calling `return_to_nix`, an error is raised. + +Standard output and standard error from the WASI module are captured and emitted as Nix warnings (one warning per line). + +## Host Functions + +All host functions are imported from the `env` module. + +### Error Handling + +#### `panic(ptr: u32, len: u32)` + +Aborts execution with an error message. + +**Parameters:** +- `ptr` - Pointer to UTF-8 encoded error message in Wasm memory +- `len` - Length of the error message in bytes + +#### `warn(ptr: u32, len: u32)` + +Emits a warning message. + +**Parameters:** +- `ptr` - Pointer to UTF-8 encoded warning message in Wasm memory +- `len` - Length of the warning message in bytes + +### Type Inspection + +#### `get_type(value: ValueId) -> u32` + +Returns the type of a Nix value. + +**Parameters:** +- `value` - ID of a Nix value + +**Return values:** +- `1` - Integer +- `2` - Float +- `3` - Boolean +- `4` - String +- `5` - Path +- `6` - Null +- `7` - Attribute set +- `8` - List +- `9` - Function + +**Note:** Forces evaluation of the value. + +### Integer Operations + +#### `make_int(n: i64) -> ValueId` + +Creates a Nix integer value. + +**Parameters:** +- `n` - The integer value + +**Returns:** Value ID of the created integer + +#### `get_int(value: ValueId) -> i64` + +Extracts an integer from a Nix value. Throws an error if the value is not an integer. + +**Parameters:** +- `value` - ID of a Nix integer value + +**Returns:** The integer value + +### Float Operations + +#### `make_float(x: f64) -> ValueId` + +Creates a Nix float value. + +**Parameters:** +- `x` - The float value + +**Returns:** Value ID of the created float + +#### `get_float(value: ValueId) -> f64` + +Extracts a float from a Nix value. Throws an error if the value is not a float. + +**Parameters:** +- `value` - ID of a Nix float value + +**Returns:** The float value + +### Boolean Operations + +#### `make_bool(b: i32) -> ValueId` + +Creates a Nix Boolean value. + +**Parameters:** +- `b` - Boolean value (0 = false, non-zero = true) + +**Returns:** Value ID of the created Boolean + +#### `get_bool(value: ValueId) -> i32` + +Extracts a Boolean from a Nix value. Throws an error if the value is not a Boolean. + +**Parameters:** +- `value` - ID of a Nix Boolean value + +**Returns:** 0 for false, 1 for true + +### Null Operations + +#### `make_null() -> ValueId` + +Creates a Nix null value. + +**Returns:** Value ID of the null value + +### String Operations + +#### `make_string(ptr: u32, len: u32) -> ValueId` + +Creates a Nix string value from Wasm memory. + +**Parameters:** +- `ptr` - Pointer to a string in Wasm memory +- `len` - Length of the string in bytes + +**Note:** Strings do not require a null terminator. + +**Returns:** Value ID of the created string + +#### `copy_string(value: ValueId, ptr: u32, max_len: u32) -> u32` + +Copies a Nix string value into Wasm memory. + +**Parameters:** +- `value` - ID of a string value +- `ptr` - Pointer to buffer in Wasm memory +- `max_len` - Maximum number of bytes to copy + +**Returns:** The actual length of the string in bytes + +**Note:** If the returned length is greater than `max_len`, no data is copied. Call again with a larger buffer to get the full string. + +### Path Operations + +#### `make_path(base: ValueId, ptr: u32, len: u32) -> ValueId` + +Creates a Nix path value relative to a base path. + +**Parameters:** +- `base` - ID of a path value +- `ptr` - Pointer to a string in Wasm memory +- `len` - Length of the path string in bytes + +**Returns:** ID of a new path value + +**Note:** The path string is interpreted relative to the base path. The resulting path is in the same source tree ("source accessor") as the original path. + +#### `copy_path(value: ValueId, ptr: u32, max_len: u32) -> u32` + +Copies a Nix path value into Wasm memory as an absolute path string. + +**Parameters:** +- `value` - ID of a path value +- `ptr` - Pointer to buffer in Wasm memory +- `max_len` - Maximum number of bytes to copy + +**Returns:** The actual length of the path string in bytes + +**Note:** If the returned length is greater than `max_len`, no data is copied. + +### List Operations + +#### `make_list(ptr: u32, len: u32) -> ValueId` + +Creates a Nix list from an array of value IDs in Wasm memory. + +**Parameters:** +- `ptr` - Pointer to array of `ValueId` (u32) in Wasm memory +- `len` - Number of elements in the array + +**Returns:** Value ID of the created list + +**Note:** The array must contain `len * 4` bytes (each ValueId is 4 bytes). + +#### `copy_list(value: ValueId, ptr: u32, max_len: u32) -> u32` + +Copies a Nix list into Wasm memory as an array of value IDs. + +**Parameters:** +- `value` - ID of a list value +- `ptr` - Pointer to buffer in Wasm memory +- `max_len` - Maximum number of elements to copy + +**Returns:** The actual number of elements in the list + +**Note:** If the returned length is greater than `max_len`, no data is copied. Each element is written as a `ValueId` (4 bytes). The buffer must be `max_len * 4` bytes large. + +### Attribute Set Operations + +#### `make_attrset(ptr: u32, len: u32) -> ValueId` + +Creates a Nix attribute set from an array of attributes in Wasm memory. + +**Parameters:** +- `ptr` - Pointer to array of attribute structures in Wasm memory +- `len` - Number of attributes + +**Returns:** Value ID of the created attribute set + +**Attribute structure format:** +```c +struct Attr { + name_ptr: u32, // Pointer to attribute name + name_len: u32, // Length of attribute name in bytes + value_id: u32, // ID of the attribute value +} +``` + +Each `Attr` element is 12 bytes (3 × 4 bytes). + +#### `copy_attrset(value: ValueId, ptr: u32, max_len: u32) -> u32` + +Copies a Nix attribute set into Wasm memory as an array of attribute structures. + +**Parameters:** +- `value` - ID of a Nix attribute set value +- `ptr` - Pointer to buffer in Wasm memory +- `max_len` - Maximum number of attributes to copy + +**Returns:** The actual number of attributes in the set + +**Note:** If the returned length is greater than `max_len`, no data is copied. + +**Output structure format:** +```c +struct Attr { + value_id: u32, // ID of the attribute value + name_len: u32, // Length of attribute name in bytes +} +``` + +Each attribute is 8 bytes (2 × 4 bytes). Use `copy_attrname` to retrieve attribute names. + +#### `copy_attrname(value: ValueId, attr_idx: u32, ptr: u32, len: u32)` + +Copies an attribute name into Wasm memory. + +**Parameters:** +- `value` - ID of a Nix attribute set value +- `attr_idx` - Index of the attribute (from `copy_attrset`) +- `ptr` - Pointer to buffer in Wasm memory +- `len` - Length of the buffer (must exactly match the attribute name length) + +**Note:** Throws an error if `len` doesn't match the attribute name length or if `attr_idx` is out of bounds. + +#### `get_attr(value: ValueId, ptr: u32, len: u32) -> ValueId` + +Gets an attribute value from an attribute set by name. + +**Parameters:** +- `value` - ID of a Nix attribute set value +- `ptr` - Pointer to the attribute name in Wasm memory +- `len` - Length of the attribute name in bytes + +**Returns:** Value ID of the attribute value, or 0 if the attribute doesn't exist + +### Function Operations + +#### `call_function(fun: ValueId, ptr: u32, len: u32) -> ValueId` + +Calls a Nix function with arguments. + +**Parameters:** +- `fun` - ID of a Nix function value +- `ptr` - Pointer to array of `ValueId` arguments in Wasm memory +- `len` - Number of arguments + +**Returns:** Value ID of the function result + +#### `make_app(fun: ValueId, ptr: u32, len: u32) -> ValueId` + +Creates a lazy or partially applied function application. + +**Parameters:** +- `fun` - ID of a Nix function value +- `ptr` - Pointer to array of `ValueId` arguments in Wasm memory +- `len` - Number of arguments + +**Returns:** Value ID of the unevaluated application + +### Returning Results (WASI mode only) + +#### `return_to_nix(value: ValueId)` + +Returns a result value to the Nix evaluator from a WASI module. This function is only available in WASI mode. + +**Parameters:** +- `value` - ID of the Nix value to return as the result of the `builtins.wasm` call + +**Note:** Calling this function immediately terminates the WASI module's execution. The module must call `return_to_nix` before finishing; otherwise, an error is raised. + +### File I/O + +#### `read_file(path: ValueId, ptr: u32, len: u32) -> u32` + +Reads a file into Wasm memory. + +**Parameters:** +- `path` - Value ID of a Nix path value +- `ptr` - Pointer to buffer in Wasm memory +- `len` - Maximum number of bytes to read + +**Returns:** The actual file size in bytes + +**Note:** Similar to `builtins.readFile`, but can handle files that cannot be represented as Nix strings (in particular, files containing NUL bytes). If the returned size is greater than `len`, no data is copied. + +## Example Usage + +For Rust bindings to this interface and several examples, see https://github.com/DeterminateSystems/nix-wasm-rust/. diff --git a/doc/manual/source/quick-start.md b/doc/manual/source/quick-start.md index 9eb7a3265903..42e4e9c0c247 100644 --- a/doc/manual/source/quick-start.md +++ b/doc/manual/source/quick-start.md @@ -3,10 +3,13 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix: +1. Install Nix. + We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. + For Linux and Windows Subsystem for Linux (WSL) users: ```console - $ curl -L https://nixos.org/nix/install | sh + $ curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install ``` The install script will use `sudo`, so make sure you have sufficient rights. @@ -41,3 +44,5 @@ For more in-depth information you are kindly referred to subsequent chapters. ```console $ nix-collect-garbage ``` + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md new file mode 100644 index 000000000000..f557e8f7fc65 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -0,0 +1,226 @@ +# Changes between Nix and Determinate Nix + +This section lists the differences between upstream Nix 2.34 and Determinate Nix 3.21.9. + +* In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. + +* In Determinate Nix, the new Nix CLI (i.e. the `nix` command) is stable. You no longer need to enable the `nix-command` experimental feature. + +* Determinate Nix has a setting [`json-log-path`](@docroot@/command-ref/conf-file.md#conf-json-log-path) to send a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. + +* Determinate Nix has made `nix profile install` an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. + +* `nix-channel` and `channel:` url syntax (like `channel:nixos-24.11`) is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/34 + +* Using indirect flake references and implicit inputs is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/37 + +* Warnings around "dirty trees" are updated to reduce "dirty" jargon, and now refers to "uncommitted changes". + + + + + + + +* `nix upgrade-nix` is now inert, and suggests using `determinate-nixd upgrade`. [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) + +* Determinate Nix has Lazy Trees, avoiding expensive copying of flake inputs to the Nix store. ([DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27), [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56)) + + + + + + + + + +* Documentation on how to replicate `nix-store --query --deriver` with the new `nix` cli. [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) + +* In `nix profile`, the symbols `ε` and `∅` have been replaced with descriptive English words. [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) + + + + + + + +* When remote building with `--keep-failed`, Determinate Nix shows "you can rerun" message if the derivation's platform is supported on this machine. [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) + +* Improved error message when `sandbox-paths` specifies a missing file. [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) + + + + + + + + + +* `nix store delete` now explains why deletion fails. [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + + + + + + + + + + + + + +* Tab completing arguments to Nix avoids network access. [DeterminateSystems/nix-src#161](https://github.com/DeterminateSystems/nix-src/pull/161) + +* Importing Nixpkgs and other tarballs to the cache is 2-4x faster. [DeterminateSystems/nix-src#149](https://github.com/DeterminateSystems/nix-src/pull/149) + +* Adding paths to the store is significantly faster. [DeterminateSystems/nix-src#162](https://github.com/DeterminateSystems/nix-src/pull/162) + + + + + +* Determinate Nix allows flake inputs to be fetched at build time. [DeterminateSystems/nix-src#49](https://github.com/DeterminateSystems/nix-src/pull/49) + + + +* The default `nix flake init` template is much more useful. [DeterminateSystems/nix-src#180](https://github.com/DeterminateSystems/nix-src/pull/180) + + + + + + + + +* Multithreaded evaluation support. [DeterminateSystems/nix-src#125](https://github.com/DeterminateSystems/nix-src/pull/125) + + + + + + +* Determinate Nix only tries to substitute inputs if fetching from its original location fails.[DeterminateSystems/nix-src#202](https://github.com/DeterminateSystems/nix-src/pull/202) + + + + + + +* A new command `nix nario` that replaces `nix-store --export|--export`. It also has a new file format (`--format 2`) that supports store path attributes such as signatures, and that can be imported more efficiently. [DeterminateSystems/nix-src#215](https://github.com/DeterminateSystems/nix-src/pull/215) + +* Determinate Nix prints the Nix version when using `-vv` or higher verbosity. [DeterminateSystems/nix-src#237](https://github.com/DeterminateSystems/nix-src/pull/237) + + + + +* During evaluation, you can read or import from the result of `builtins.fetchClosure`. [DeterminateSystems/nix-src#241](https://github.com/DeterminateSystems/nix-src/pull/241) + + + +* Flakerefs in error messages and lockfile diffs are abbreviated for readability. [DeterminateSystems/nix-src#243](https://github.com/DeterminateSystems/nix-src/pull/243), [DeterminateSystems/nix-src#264](https://github.com/DeterminateSystems/nix-src/pull/264) + + + + + + + + +* The Git fetcher doesn't compute `revCount` or `lastModified` if they're already specified [DeterminateSystems./nix-src#269](https://github.com/DeterminateSystems/nix-src/pull/269) + +* The Git fetcher avoids doing a shallow Git fetch if it previously did a non-shallow fetch of the same repository. [DeterminateSystems/nix-src#270](https://github.com/DeterminateSystems/nix-src/pull/270) + +* Determinate Nix has a builtin copy of the flake registry, making it more resilient to network outages. [DeterminateSystems/nix-src#271](https://github.com/DeterminateSystems/nix-src/pull/271) + + + +* `nix build` and `nix profile` report failing or succeeding installables. [DeterminateSystems/nix-src#281](https://github.com/DeterminateSystems/nix-src/pull/281) + +* `nix flake check` shows which outputs failed or succeeded. [DeterminateSystems/nix-src#285](https://github.com/DeterminateSystems/nix-src/pull/285) + +* Determinate Nix has a `nix ps` command to show active builds. [DeterminateSystems/nix-src#282](https://github.com/DeterminateSystems/nix-src/pull/282) + +* Determinate Nix has improved backward compatibility with lock files created by Nix < 2.20. [DeterminateSystems/nix-src#278](https://github.com/DeterminateSystems/nix-src/pull/278) + + + +* Determinate Nix has a builtin function `builtins.filterAttrs`. [DeterminateSystems/nix-src#291](https://github.com/DeterminateSystems/nix-src/pull/291) + +* `builtins.fetchTree` implicitly sets `__final = true` when a `narHash` is supplied. This allows the tree to be substituted. [DeterminateSystems/nix-src#297](https://github.com/DeterminateSystems/nix-src/pull/297) + + + + + + + +* Determinate Nix has an experimental builtin `builtins.wasm` that allows the Nix language to be extended using any language that compiles to Wasm. [DeterminateSystems/nix-src#309](https://github.com/DeterminateSystems/nix-src/pull/309) + +* `builtins.getFlake` supports path values. [DeterminateSystems/nix-src#338](https://github.com/DeterminateSystems/nix-src/pull/338) + +* Determinate Nix has support for keeping track of the provenance of store paths. [DeterminateSystems/nix-src#321](https://github.com/DeterminateSystems/nix-src/pull/321) + + + + + + + + + + + + + + + + + + + + +* Determinate Nix can upload crash info to Sentry. [DeterminateSystems/nix-src#418](https://github.com/DeterminateSystems/nix-src/pull/418) + +* Determinate Nix provides the pre-build hook with a JSON serialization of the derivation. [DeterminateSystems/nix-src#424](https://github.com/DeterminateSystems/nix-src/pull/424) + + + + + + + + + + +* Determinate Nix supports resuming from binary caches that don't support ranged requests. [DeterminateSystems/nix-src#445](https://github.com/DeterminateSystems/nix-src/pull/445) + +* JSON log messages have a configurable session ID field. [DeterminateSystems/nix-src#440](https://github.com/DeterminateSystems/nix-src/pull/440) + + + +* Determinate Nix has a command `nix serve` that allows any store to be served as a binary cache. [DeterminateSystems/nix-src#428](https://github.com/DeterminateSystems/nix-src/pull/428) + +* Determinate Nix has experimental support for signing store paths using CNSA algorithms. [DeterminateSystems/nix-src#449](https://github.com/DeterminateSystems/nix-src/pull/449) + + + +* Determinate Nix is built against Determinate Secure Packages. [DeterminateSystems/nix-src#288](https://github.com/DeterminateSystems/nix-src/pull/288) + + + +* `nix optimise store` is multi-threaded in Determinate Nix. [DeterminateSystems/nix-src#492](https://github.com/DeterminateSystems/nix-src/pull/492) + + + + + + + + + + + + + + diff --git a/doc/manual/source/release-notes-determinate/index.md b/doc/manual/source/release-notes-determinate/index.md new file mode 100644 index 000000000000..bba33084424c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/index.md @@ -0,0 +1,3 @@ +# Determinate Nix Release Notes + +This chapter lists the differences between Nix and Determinate Nix, as well as the release history of Determinate Nix. diff --git a/doc/manual/source/release-notes-determinate/rl-3.0.0.md b/doc/manual/source/release-notes-determinate/rl-3.0.0.md new file mode 100644 index 000000000000..d60786e9a72f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.0.0.md @@ -0,0 +1,5 @@ +# Release 3.0.0 (2025-03-04) + +* Initial release of Determinate Nix. + +* Based on [upstream Nix 2.26.2](../release-notes/rl-2.26.md). diff --git a/doc/manual/source/release-notes-determinate/rl-3.1.0.md b/doc/manual/source/release-notes-determinate/rl-3.1.0.md new file mode 100644 index 000000000000..96b7819d08db --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.1.0.md @@ -0,0 +1,5 @@ +# Release 3.1.0 (2025-03-27) + +* Based on [upstream Nix 2.27.1](../release-notes/rl-2.27.md). + +* New setting `json-log-path` that sends a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. diff --git a/doc/manual/source/release-notes-determinate/rl-3.3.0.md b/doc/manual/source/release-notes-determinate/rl-3.3.0.md new file mode 100644 index 000000000000..badf96415df0 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.3.0.md @@ -0,0 +1,5 @@ +# Release 3.3.0 (2025-04-11) + +* Based on [upstream Nix 2.28.1](../release-notes/rl-2.28.md). + +* The `nix profile install` command is now an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.0.md b/doc/manual/source/release-notes-determinate/rl-3.4.0.md new file mode 100644 index 000000000000..24ae03ca554f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.0.md @@ -0,0 +1,50 @@ +# Release 3.4.0 (2025-04-25) + +* Based on [upstream Nix 2.28.2](../release-notes/rl-2.28.md). + +* **Warn users that `nix-channel` is deprecated.** + +This is the first change accomplishing our roadmap item of deprecating Nix channels: https://github.com/DeterminateSystems/nix-src/issues/34 + +This is due to user confusion and surprising behavior of channels, especially in the context of user vs. root channels. + +The goal of this change is to make the user experience of Nix more predictable. +In particular, these changes are to support users with lower levels of experience who are following guides that focus on channels as the mechanism of distribution. + +Users will now see this message: + +> nix-channel is deprecated in favor of flakes in Determinate Nix. For a guide on Nix flakes, see: https://zero-to-nix.com/. or details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + + +* **Warn users that `channel:` URLs are deprecated.** + +This is the second change regarding our deprecation of Nix channels. +Using a `channel:` URL (like `channel:nixos-24.11`) will yield a warning like this: + +> Channels are deprecated in favor of flakes in Determinate Nix. Instead of 'channel:nixos-24.11', use 'https://nixos.org/channels/nixos-24.11/nixexprs.tar.xz'. For a guide on Nix flakes, see: https://zero-to-nix.com/. For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + +* **Warn users against indirect flake references in `flake.nix` inputs** + +This is the first change accomplishing our roadmap item of deprecating implicit and indirect flake inputs: https://github.com/DeterminateSystems/nix-src/issues/37 + +The flake registry provides an important UX affordance for using Nix flakes and remote sources in command line uses. +For that reason, the registry is not being deprecated entirely and will still be used for command-line incantations, like nix run. + +This move will eliminate user confusion and surprising behavior around global and local registries during flake input resolution. + +The goal of this change is to make the user experience of Nix more predictable. +We have seen a pattern of confusion when using automatic flake inputs and local registries. +Specifically, users' flake inputs resolving and locking inconsistently depending on the configuration of the host system. + +Users will now see the following warning if their flake.nix uses an implicit or indirect Flake reference input: + +> Flake input 'nixpkgs' uses the flake registry. Using the registry in flake inputs is deprecated in Determinate Nix. To make your flake future-proof, add the following to 'xxx/flake.nix': +> +> inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; +> +> For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37 + + +### Other updates: +* Improve the "dirty tree" message. Determinate Nix will now say `Git tree '...' has uncommitted changes` instead of `Git tree '...' is dirty` +* Stop warning about uncommitted changes in a Git repository when using `nix develop` diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.2.md b/doc/manual/source/release-notes-determinate/rl-3.4.2.md new file mode 100644 index 000000000000..8acabd4425fd --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.2.md @@ -0,0 +1,4 @@ +# Release 3.4.2 (2025-05-05) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.0.md b/doc/manual/source/release-notes-determinate/rl-3.5.0.md new file mode 100644 index 000000000000..d5b26b9419e7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.0.md @@ -0,0 +1,4 @@ +# Release 3.5.0 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.1.md b/doc/manual/source/release-notes-determinate/rl-3.5.1.md new file mode 100644 index 000000000000..b0813ca59c90 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.1.md @@ -0,0 +1,57 @@ +# Release 3.5.1 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed + +Most notably, Lazy Trees has merged in to Determinate Nix and is in Feature Preview status, but remains disabled by default. +Lazy trees massively improves performance in virtually all scenarios because it enables Nix to avoid making unnecessary copies of files into the Nix store. +In testing, we saw iteration times on Nixpkgs **drop from over 12 seconds to 3.5 seconds**. + +After upgrading to Determinate Nix 3.5.1 with `sudo determinate-nixd upgrade`, enable lazy trees by adding this to `/etc/nix/nix.custom.conf`: + +``` +lazy-trees = true +``` + +Please note that our full flake regression test suite passes with no changes with lazy trees, and please report compatibility issues. + +Read [this GitHub comment](https://github.com/DeterminateSystems/nix-src/pull/27#pullrequestreview-2822153088) for further details and next steps. +We'll be publishing an update on the [Determinate Systems blog](https://determinate.systems/posts/) in the next few days with more information as well. + +Relevant PRs: +* Lazy trees v2 by @edolstra in [DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27) +* Improve lazy trees backward compatibility by @edolstra in [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56) + + +### Additional changes in this release: +* Bug fix: Flake input URLs are canonicalized before checking flake.lock file staleness, avoiding needlessly regenerating flake.lock files with `dir` in URL-style flakerefs by @edolstra in [DeterminateSystems/nix-src#57](https://github.com/DeterminateSystems/nix-src/pull/57) +* `nix upgrade-nix` is deprecated in favor of `determinate-nixd upgrade`, by @gustavderdrache in [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) +* UX: Improved build failure and dependency failure error messages to include needed output paths by @edolstra in [DeterminateSystems/nix-src#58](https://github.com/DeterminateSystems/nix-src/pull/58). + +Previously: + +``` +error: builder for '/nix/store/[...]-nested-failure-bottom.drv' failed with exit code 1 +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-middle.drv' failed to build +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-top.drv' failed to build +``` + +Now: + +``` +error: Cannot build '/nix/store/w37gflm9wz9dcnsgy3sfrmnlvm8qigaj-nested-failure-bottom.drv'. + Reason: builder failed with exit code 1. + Output paths: + /nix/store/yzybs8kp35dfipbzdlqcc6lxz62hax04-nested-failure-bottom +error: Cannot build '/nix/store/00gr5hlxfc03x2675w6nn3pwfrz2fr62-nested-failure-middle.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/h781j5h4bdchmb4c2lvy8qzh8733azhz-nested-failure-middle +error: Cannot build '/nix/store/8am0ng1gyx8sbzyr0yx6jd5ix3yy5szc-nested-failure-top.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/fh12637kgvp906s9yhi9w2dc7ghfwxs1-nested-failure-top +``` + +**Full Changelog**: [v3.4.2...v3.5.1](https://github.com/DeterminateSystems/nix-src/compare/v3.4.2...v3.5.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.2.md b/doc/manual/source/release-notes-determinate/rl-3.5.2.md new file mode 100644 index 000000000000..bc5396c255b6 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.2.md @@ -0,0 +1,11 @@ +# Release 3.5.2 (2025-05-12) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed +* Fix a regression where narHash was not added to lock files when lazy trees were disabled by @edolstra in [DeterminateSystems/nix-src#63](https://github.com/DeterminateSystems/nix-src/pull/63) + +* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) + + +**Full Changelog**: [v3.5.1...v3.5.2](https://github.com/DeterminateSystems/nix-src/compare/v3.5.1...v3.5.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.0.md b/doc/manual/source/release-notes-determinate/rl-3.6.0.md new file mode 100644 index 000000000000..453ab6c301dc --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.0.md @@ -0,0 +1,11 @@ +# Release 3.6.0 (2025-05-22) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) +* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + +**Full Changelog**: [v3.5.2...v3.6.0](https://github.com/DeterminateSystems/nix-src/compare/v3.5.2...v3.6.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.1.md b/doc/manual/source/release-notes-determinate/rl-3.6.1.md new file mode 100644 index 000000000000..12505afee278 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.1.md @@ -0,0 +1,9 @@ +# Release 3.6.1 (2025-05-24) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Fix nlohmann error in fromStructuredAttrs() by @edolstra in [DeterminateSystems/nix-src#73](https://github.com/DeterminateSystems/nix-src/pull/73) + + +**Full Changelog**: [v3.6.0...v3.6.1](https://github.com/DeterminateSystems/nix-src/compare/v3.6.0...v3.6.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.2.md b/doc/manual/source/release-notes-determinate/rl-3.6.2.md new file mode 100644 index 000000000000..882c142f00c3 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.2.md @@ -0,0 +1,15 @@ +# Release 3.6.2 (2025-06-02) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Dramatically improve the performance of nix store copy-sigs: Use http-connections setting to control parallelism by @edolstra in [DeterminateSystems/nix-src#80](https://github.com/DeterminateSystems/nix-src/pull/80) +* Document how to replicate nix-store --query --deriver with the nix cli by @grahamc in [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) +* The garbage collector no longer gives up if it encounters an undeletable file, by @edolstra in [DeterminateSystems/nix-src#83](https://github.com/DeterminateSystems/nix-src/pull/83) +* nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) +* Rework README to clarify that this distribution is our distribution, by @lucperkins in [DeterminateSystems/nix-src#84](https://github.com/DeterminateSystems/nix-src/pull/84) +* Include the source location when warning about inefficient double copies by @edolstra in [DeterminateSystems/nix-src#79](https://github.com/DeterminateSystems/nix-src/pull/79) +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + +**Full Changelog**: [v3.6.1...v3.6.2](https://github.com/DeterminateSystems/nix-src/compare/v3.6.1...v3.6.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.5.md b/doc/manual/source/release-notes-determinate/rl-3.6.5.md new file mode 100644 index 000000000000..8ef5be0fd0d3 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.5.md @@ -0,0 +1,19 @@ +# Release 3.6.5 (2025-06-12) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) +* Improve caching of inputs by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98), [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110), and [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + +**Full Changelog**: [v3.6.2...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md new file mode 100644 index 000000000000..bf4e3690afa1 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -0,0 +1,7 @@ +# Release 3.6.6 (2025-06-17) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed + +* No-op release on the nix-src side, due to a regression on nix-darwin in determinate-nixd. diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md new file mode 100644 index 000000000000..197587f1b3a9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -0,0 +1,17 @@ +# Release 3.6.7 (2025-06-24) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Security contents + +* Patched against GHSA-g948-229j-48j3 + +### Lazy trees: + +* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + + +**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md new file mode 100644 index 000000000000..c4b4b96c9e73 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.8.md @@ -0,0 +1,12 @@ +# Release 3.6.8 (2025-06-25) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + +**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8) diff --git a/doc/manual/source/release-notes-determinate/rl-3.7.0.md b/doc/manual/source/release-notes-determinate/rl-3.7.0.md new file mode 100644 index 000000000000..615e858592e2 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.7.0.md @@ -0,0 +1,63 @@ +# Release 3.7.0 (2025-07-03) + +- Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Prefetch flake inputs in parallel + +By @edolstra in [DeterminateSystems/nix-src#127](https://github.com/DeterminateSystems/nix-src/pull/127) + +This release brings the command `nix flake prefetch-inputs`. + +Flake inputs are typically fetched "just in time." +That means Nix fetches a flake input when the evaluator needs it, and not before. +When the evaluator needs an input, evaluation is paused until the source is available. + +This causes a significant slow-down on projects with lots of flake inputs. + +The new command `nix flake prefetch-inputs` fetches all flake inputs in parallel. +We expect running this new command before building will dramatically improve evaluation performance for most projects, especially in CI. +Note that projects which with many unused flake inputs may not benefit from this change, since the new command fetches every input whether they're used or not. + +### Deep flake input overrides now work as expected + +By @edolstra in [DeterminateSystems/nix-src#108](https://github.com/DeterminateSystems/nix-src/pull/108) + +An override like: + +``` +inputs.foo.inputs.bar.inputs.nixpkgs.follows = "nixpkgs"; +``` + +implicitly set `inputs.foo.inputs.bar` to `flake:bar`, which led to an unexpected error like: + +``` +error: cannot find flake 'flake:bar' in the flake registries +``` + +We now no longer create a parent override (like for `foo.bar` in the example above) if it doesn't set an explicit ref or follows attribute. +We only recursively apply its child overrides. + +### `nix store delete` now shows you why deletion was not possible + +By @edolstra in [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + +For example: + +``` +error: Cannot delete path '/nix/store/6fcrjgfjip2ww3sx51rrmmghfsf60jvi-patchelf-0.14.3' + because it's referenced by the GC root '/home/eelco/Dev/nix-master/build/result'. + +error: Cannot delete path '/nix/store/lf3lrf8bjfn8xvr0az9q96y989sxs5r9-cowsay-3.8.4' + because it's referenced by the GC root '/proc/3600568/environ'. + +error: Cannot delete path '/nix/store/klyng5rpdkwi5kbxkncy4gjwb490dlhb-foo.drv' + because it's in use by '{nix-process:3605324}'. +``` + +### Lazy-tree improvements + +- Improved lazy-tree evaluation caching for flakes accessed with a `path` flakeref by @edolstra in [DeterminateSystems/nix-src#131](https://github.com/DeterminateSystems/nix-src/pull/131) + +**Full Changelog**: [v3.6.8...v3.7.0](https://github.com/DeterminateSystems/nix-src/compare/v3.6.8...v3.7.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.0.md b/doc/manual/source/release-notes-determinate/rl-3.8.0.md new file mode 100644 index 000000000000..4103d6df94e0 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.0.md @@ -0,0 +1,29 @@ +# Release 3.8.0 (2025-07-10) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed + +### Faster CI with `nix flake check` + +`nix flake check` no longer downloads flake outputs if no building is necessary. + +This command is intended to validate that a flake can fully evaluate and all outputs can build. +If the outputs are available in a binary cache then both properties are confirmed to be true. +Notably, downloading the output from the binary cache is not strictly necessary for the validation. + +Previously, `nix flake check` would download a flake output if the full build is available in a binary cache. + +Some users will find this change significantly reduces costly bandwidth and CI workflow time. + +PR: [DeterminateSystems/nix-src#134](https://github.com/DeterminateSystems/nix-src/pull/134) + +### Improved flake locking of transitive dependencies + +Determinate Nix now re-locks all transitive dependencies when changing a flake input's source URL. + +This fixes an issue where in some scenarios Nix would not re-lock those inputs and incorrectly use the old inputs' dependencies. + +PR: [DeterminateSystems/nix-src#137](https://github.com/DeterminateSystems/nix-src/pull/137) + +**Full Changelog**: [v3.7.0...v3.8.0](https://github.com/DeterminateSystems/nix-src/compare/v3.7.0...v3.8.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.1.md b/doc/manual/source/release-notes-determinate/rl-3.8.1.md new file mode 100644 index 000000000000..90dc328f6ec2 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.1.md @@ -0,0 +1,9 @@ +# Release 3.8.1 (2025-07-11) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* Address ifdef problem with macOS/BSD sandboxing by @gustavderdrache in [DeterminateSystems/nix-src#142](https://github.com/DeterminateSystems/nix-src/pull/142) + + +**Full Changelog**: [v3.8.0...v3.8.1](https://github.com/DeterminateSystems/nix-src/compare/v3.8.0...v3.8.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.2.md b/doc/manual/source/release-notes-determinate/rl-3.8.2.md new file mode 100644 index 000000000000..638d90f6841b --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.2.md @@ -0,0 +1,10 @@ +# Release 3.8.2 (2025-07-12) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* ci: don't run the full test suite for x86_64-darwin by @grahamc in [DeterminateSystems/nix-src#144](https://github.com/DeterminateSystems/nix-src/pull/144) +* Try publishing the manual again by @grahamc in [DeterminateSystems/nix-src#145](https://github.com/DeterminateSystems/nix-src/pull/145) + + +**Full Changelog**: [v3.8.1...v3.8.2](https://github.com/DeterminateSystems/nix-src/compare/v3.8.1...v3.8.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.3.md b/doc/manual/source/release-notes-determinate/rl-3.8.3.md new file mode 100644 index 000000000000..d3eb02bc7ea5 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.3.md @@ -0,0 +1,26 @@ +# Release 3.8.3 (2025-07-18) + +* Based on [upstream Nix 2.30.1](../release-notes/rl-2.30.md). + +## What's Changed + +### Non-blocking evaluation caching + +Users reported evaluation would occasionally block other evaluation processes. + +The evaluation cache database is now opened in write-ahead mode to prevent delaying evaluations. + +PR: [DeterminateSystems/nix-src#150](https://github.com/DeterminateSystems/nix-src/pull/150) + +### New experimental feature: `external-builders` + +This experimental feature allows Nix to call an external program for the build environment. + +The interface and behavior of this feature may change at any moment without a correspondingly major semver version change. + +PRs: +- [DeterminateSystems/nix-src#141](https://github.com/DeterminateSystems/nix-src/pull/141) +- [DeterminateSystems/nix-src#152](https://github.com/DeterminateSystems/nix-src/pull/152) +- [DeterminateSystems/nix-src#78](https://github.com/DeterminateSystems/nix-src/pull/78) + +**Full Changelog**: [v3.8.2...v3.8.3](https://github.com/DeterminateSystems/nix-src/compare/v3.8.2...v3.8.3) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.4.md b/doc/manual/source/release-notes-determinate/rl-3.8.4.md new file mode 100644 index 000000000000..7c73e75ca023 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.4.md @@ -0,0 +1,9 @@ +# Release 3.8.4 (2025-07-21) + +* Based on [upstream Nix 2.30.1](../release-notes/rl-2.30.md). + +## What's Changed +* Revert "Use WAL mode for SQLite cache databases" by @grahamc in [DeterminateSystems/nix-src#155](https://github.com/DeterminateSystems/nix-src/pull/155) + + +**Full Changelog**: [v3.8.3...v3.8.4](https://github.com/DeterminateSystems/nix-src/compare/v3.8.3...v3.8.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.5.md b/doc/manual/source/release-notes-determinate/rl-3.8.5.md new file mode 100644 index 000000000000..0f1bbe6f99d7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.5.md @@ -0,0 +1,58 @@ +## What's Changed + +### Less time "unpacking into the Git cache" + +Unpacking sources into the user's cache is now takes 1/2 to 1/4 of the time it used to. +Previously, Nix serially unpacked sources into the cache. +This change takes better advantage of our users' hardware by parallelizing the import. +Real life testing shows an initial Nixpkgs import takes 3.6s on Linux, when it used to take 11.7s. + +PR: [DeterminateSystems/nix-src#149](https://github.com/DeterminateSystems/nix-src/pull/149) + +### Copy paths to the daemon in parallel + +Determinate Nix's evaluator no longer blocks evaluation when copying paths to the store. +Previously, Nix would pause evaluation when it needed to add files to the store. +Now, the copying is performed in the background allowing evaluation to proceed. + +PR: [DeterminateSystems/nix-src#162](https://github.com/DeterminateSystems/nix-src/pull/162) + +### Faster Nix evaluation by reducing duplicate Nix daemon queries + +Determinate Nix more effectively caches store path validity data within a single evaluation. +Previously, the Nix client would perform many thousands of exra Nix daemon requests. +Each extra request takes real time, and this change reduced a sample evaluation by over 12,000 requests. + +PR: [DeterminateSystems/nix-src#157](https://github.com/DeterminateSystems/nix-src/pull/157) + +### More responsive tab completion + +Tab completion now implies the "--offline" flag, which disables most network requests. +Previously, tab completing Nix arguments would attempt to fetch sources and access binary caches. +Operating in offline mode improves the interactive experience of Nix when tab completing. + +PR: [DeterminateSystems/nix-src#161](https://github.com/DeterminateSystems/nix-src/pull/161) + +### ZFS users: we fixed the mysterious stall. + +Opening the Nix database is usually instantaneous but sometimes has a several second latency. +Determinate Nix works around this issue, eliminating the frustrating random stall when running Nix commands. + +PR: [DeterminateSystems/nix-src#158](https://github.com/DeterminateSystems/nix-src/pull/158) + +### Other changes + +* Determinate Nix is now fully formatted by clang-format, making it easier than ever to contribute to the project. + +PR: [DeterminateSystems/nix-src#159](https://github.com/DeterminateSystems/nix-src/pull/159) + +* Determinate Nix is now based on upstream Nix 2.30.2. + +PR: [DeterminateSystems/nix-src#160](https://github.com/DeterminateSystems/nix-src/pull/160) + +* Determinate Nix now uses `main` as our development branch, moving away from `detsys-main`. + +PRs: +* [DeterminateSystems/nix-src#164](https://github.com/DeterminateSystems/nix-src/pull/164) +* [DeterminateSystems/nix-src#166](https://github.com/DeterminateSystems/nix-src/pull/166) + diff --git a/doc/manual/source/release-notes-determinate/v3.10.0.md b/doc/manual/source/release-notes-determinate/v3.10.0.md new file mode 100644 index 000000000000..c644dd787446 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.10.0.md @@ -0,0 +1,10 @@ +# Release 3.10.0 (2025-09-02) + +* Based on [upstream Nix 2.31.0](../release-notes/rl-2.31.md). + +## What's Changed + +This release rebases Determinate Nix on upstream Nix 2.31.0. + + +**Full Changelog**: [v3.9.1...v3.10.0](https://github.com/DeterminateSystems/nix-src/compare/v3.9.1...v3.10.0) diff --git a/doc/manual/source/release-notes-determinate/v3.10.1.md b/doc/manual/source/release-notes-determinate/v3.10.1.md new file mode 100644 index 000000000000..08cbe4fd0583 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.10.1.md @@ -0,0 +1,9 @@ +# Release 3.10.1 (2025-09-02) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed +This release rebases Determinate Nix on upstream Nix 2.31.1. + + +**Full Changelog**: [v3.10.0...v3.10.1](https://github.com/DeterminateSystems/nix-src/compare/v3.10.0...v3.10.1) diff --git a/doc/manual/source/release-notes-determinate/v3.11.0.md b/doc/manual/source/release-notes-determinate/v3.11.0.md new file mode 100644 index 000000000000..7abb665a5a9f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.0.md @@ -0,0 +1,36 @@ +# Release 3.11.0 (2025-09-03) + +- Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed + +### Parallel evaluation + +The following commands are now able to evaluate Nix expressions in parallel: + +- `nix search` +- `nix flake check` +- `nix flake show` +- `nix eval --json` + +This is currently in developer preview, and we'll be turning it on for more users in the coming weeks. +If you would like to try it right away, specify `eval-cores` in your `/etc/nix/nix.custom.conf`: + +```ini +eval-cores = 0 # Evaluate across all cores +``` + +Further, we introduced a new builtin: `builtins.parallel`. +This new builtin allows users to explicitly parallelize evaluation within a Nix expression. + +Using this new builtin requires turning on an additional experimental feature: + +```ini +extra-experimental-features = parallel-eval +``` + +Please note that this new builtin is subject to change semantics or even go away during the developer preview. + +PR: [DeterminateSystems/nix-src#125](https://github.com/DeterminateSystems/nix-src/pull/125) + +**Full Changelog**: [v3.10.1...v3.11.0](https://github.com/DeterminateSystems/nix-src/compare/v3.10.1...v3.11.0) diff --git a/doc/manual/source/release-notes-determinate/v3.11.1.md b/doc/manual/source/release-notes-determinate/v3.11.1.md new file mode 100644 index 000000000000..305971643330 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.1.md @@ -0,0 +1,9 @@ +# Release 3.11.1 (2025-09-04) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed +* Fix race condition in Value::isTrivial() by @edolstra in [DeterminateSystems/nix-src#192](https://github.com/DeterminateSystems/nix-src/pull/192) + + +**Full Changelog**: [v3.11.0...v3.11.1](https://github.com/DeterminateSystems/nix-src/compare/v3.11.0...v3.11.1) diff --git a/doc/manual/source/release-notes-determinate/v3.11.2.md b/doc/manual/source/release-notes-determinate/v3.11.2.md new file mode 100644 index 000000000000..ac4fe569dffe --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.2.md @@ -0,0 +1,24 @@ +# Release 3.11.2 (2025-09-12) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed + +### Fix some interactions with the registry and flakes that include a `?dir=` parameter + +Some users were experiencing issues when their flake registry contained a flake that included a `?dir=` parameter, causing commands like `nix eval registry-with-flake-in-subdir#output` and those that used --inputs-from` to fail or behave incorrectly. + +This is now fixed, so use your flakes inside subdirs without fear! + +PRs: [DeterminateSystems/nix-src#196](https://github.com/DeterminateSystems/nix-src/pull/196), [DeterminateSystems/nix-src#199](https://github.com/DeterminateSystems/nix-src/pull/199) + +### Only substitute inputs if they haven't already been fetched + +When using `lazy-trees`, you might have noticed Nix fetching some source inputs from a cache, even though you could have sworn it already fetched those inputs! + +This fixes that behavior such that Nix will try to fetch inputs from their original location, and only if that fails fall back to fetching from a substituter. + +PR: [DeterminateSystems/nix-src#202](https://github.com/DeterminateSystems/nix-src/pull/202) + + +**Full Changelog**: [v3.11.1...v3.11.2](https://github.com/DeterminateSystems/nix-src/compare/v3.11.1...v3.11.2) diff --git a/doc/manual/source/release-notes-determinate/v3.11.3.md b/doc/manual/source/release-notes-determinate/v3.11.3.md new file mode 100644 index 000000000000..fab5ed51a4b5 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.3.md @@ -0,0 +1,34 @@ +# Release 3.11.3 (2025-10-09) + +* Based on [upstream Nix 2.31.2](../release-notes/rl-2.31.md). + +## What's Changed + +### Fix some bugs and interactions with parallel eval + +We received some reports of parallel eval having issues, such as not being able to be interrupted, infinite recursion hanging forever, and segfaults when using the experimental `builtins.parallel`. + +Those have now been fixed. + +Additionally, the debugger now disables parallel eval, because the two features are incompatible. + +PRs: [DeterminateSystems/nix-src#206](https://github.com/DeterminateSystems/nix-src/pull/206), [DeterminateSystems/nix-src#213](https://github.com/DeterminateSystems/nix-src/pull/213), [DeterminateSystems/nix-src#218](https://github.com/DeterminateSystems/nix-src/pull/218), [DeterminateSystems/nix-src#205](https://github.com/DeterminateSystems/nix-src/pull/205) + +### `NIX_SSHOPTS` + `ssh-ng://root@localhost` fix + +We noticed that specifying `NIX_SSHOPTS=-p2222` when using a command that uses SSH (such as `nix copy --to ssh-ng://root@localhost`) stopped respecting the `NIX_SSHOPTS` setting because of an incorrect comparison. + +This has been fixed, so `NIX_SSHOPTS` and SSH stores that are accessed like `user@localhost` work again. + +PR: [DeterminateSystems/nix-src#219](https://github.com/DeterminateSystems/nix-src/pull/219) + +### Fix `error: [json.exception.type_error.302] type must be string, but is array` when using `exportReferencesGraph` + +We received a report of a `nix build` failing on a specific flake due to its expression using `exportReferencesGraph` with a heterogeneous array of dependencies, causing this inscrutable error. + +This specific case has been broken since Nix 2.29.0, and is now fixed. + +PRs: [DeterminateSystems/nix-src#221](https://github.com/DeterminateSystems/nix-src/pull/221), [DeterminateSystems/nix-src#225](https://github.com/DeterminateSystems/nix-src/pull/225) + + +**Full Changelog**: [v3.11.2...v3.11.3](https://github.com/DeterminateSystems/nix-src/compare/v3.11.2...v3.11.3) diff --git a/doc/manual/source/release-notes-determinate/v3.12.0.md b/doc/manual/source/release-notes-determinate/v3.12.0.md new file mode 100644 index 000000000000..55c1f10bf15f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.12.0.md @@ -0,0 +1,17 @@ +# Release 3.12.0 (2025-10-23) + +* Based on [upstream Nix 2.32.1](../release-notes/rl-2.32.md). + +## What's Changed + +### `nix nario` + +Determinate Nix has a new command, `nix nario`, that replaces the commands `nix-store --export` and `nix-store --import` from the old CLI. `nix nario` allows you to serialize store paths to a file that can be imported into another Nix store. It is backwards compatible with the file format generated by `nix-store --export`. It also provides a new format (selected by passing `--format 2`) that supports store path attributes such as signatures, and allows store paths to be imported more efficiently. + +### Other changes + +`nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub. + +When using `-vv`, Determinate Nix now prints the Nix version. This is useful when diagnosing Nix problems from the debug output of a Nix run. + +**Full Changelog**: [v3.11.3...v3.12.0](https://github.com/DeterminateSystems/nix-src/compare/v3.11.3...v3.12.0) diff --git a/doc/manual/source/release-notes-determinate/v3.12.1.md b/doc/manual/source/release-notes-determinate/v3.12.1.md new file mode 100644 index 000000000000..1be2b48e26d8 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.12.1.md @@ -0,0 +1,10 @@ +# Release 3.12.1 (2025-11-04) + +* Based on [upstream Nix 2.32.1](../release-notes/rl-2.32.md). + +## What's Changed +* Allow access to the result of fetchClosure by @edolstra in [DeterminateSystems/nix-src#241](https://github.com/DeterminateSystems/nix-src/pull/241) +* libstore/build: fixup JSON logger missing the resBuildResult result event by @cole-h in [DeterminateSystems/nix-src#246](https://github.com/DeterminateSystems/nix-src/pull/246) + + +**Full Changelog**: [v3.12.0...v3.12.1](https://github.com/DeterminateSystems/nix-src/compare/v3.12.0...v3.12.1) diff --git a/doc/manual/source/release-notes-determinate/v3.12.2.md b/doc/manual/source/release-notes-determinate/v3.12.2.md new file mode 100644 index 000000000000..4c8c3169aa72 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.12.2.md @@ -0,0 +1,42 @@ +# Release 3.12.2 (2025-11-05) + +* Based on [upstream Nix 2.32.2](../release-notes/rl-2.32.md). + +## What's Changed + +### Faster `revCount` computation + +When using Git repositories with a long history, calculating the `revCount` attribute can take a long time. Determinate Nix now computes `revCount` using multiple threads, making it much faster. + +Note that if you don't need `revCount`, you can disable it altogether by setting the flake input attribute `shallow = true`. + +PR: [DeterminateSystems/nix-src#245](https://github.com/DeterminateSystems/nix-src/pull/245) + +### More readable error messages + +Previously, Nix showed full flakerefs in error messages such as stack traces, e.g. +``` + … from call site + at «github:NixOS/nixpkgs/3bea86e918d8b54aa49780505d2d4cd9261413be?narHash=sha256-Ica%2B%2BSXFuLyxX9Q7YxhfZulUif6/gwM8AEQYlUxqSgE%3D»/lib/customisation.nix:69:16: + 68| let + 69| result = f origArgs; + | ^ + 70| +``` +It now abbreviates these by leaving out `narHash` and shortening Git revisions: +``` + … from call site + at «github:NixOS/nixpkgs/3bea86e»/lib/customisation.nix:69:16: + 68| let + 69| result = f origArgs; + | ^ + 70| +``` + +PR: [DeterminateSystems/nix-src#243](https://github.com/DeterminateSystems/nix-src/pull/243) + +### Other changes + +This release fixes an assertion failure in `nix flake check`. PR: [DeterminateSystems/nix-src#252](https://github.com/DeterminateSystems/nix-src/pull/252) + +**Full Changelog**: [v3.12.1...v3.12.2](https://github.com/DeterminateSystems/nix-src/compare/v3.12.1...v3.12.2) diff --git a/doc/manual/source/release-notes-determinate/v3.13.0.md b/doc/manual/source/release-notes-determinate/v3.13.0.md new file mode 100644 index 000000000000..09041c2acda0 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.13.0.md @@ -0,0 +1,45 @@ +# Release 3.13.0 (2025-11-09) + +* Based on [upstream Nix 2.32.3](../release-notes/rl-2.32.md). + +## What's Changed + + +### Git sources have a progress indicator again + +Nix used to feel "stuck" while it was cloning large repositories. +Determinate Nix now shows git's native progress indicator while fetching. + +PR: [DeterminateSystems/nix-src#250](https://github.com/DeterminateSystems/nix-src/pull/250) + +### C API improvements + +We've invested in the C API to support our work on closure analysis for SBOM generation, and made a couple of changes: + +* C API: add nix_locked_flake_read_path for flake file reading +* C API: make nix_store_get_fs_closure compatible with upstream + +PRs: +* [DeterminateSystems/nix-src#244](https://github.com/DeterminateSystems/nix-src/pull/244) +* [DeterminateSystems/nix-src#254](https://github.com/DeterminateSystems/nix-src/pull/254) + +### Dropping support for Intel Macs + +Determinate Nix no longer supports being installed on Intel Macs. +Determinate Nix will continue to support building for Intel macOS targets, but only from an Apple Silicon host. + +From our intent-to-ship: +> Over the past year, we’ve watched usage of Determinate on Intel macOS hosts dwindle to a minuscule fraction of total usage. +> It currently stands at approximately 0.02% of all installations. +> The vast majority are run in managed CI environments that, we anticipate, will be able to easily convert to using Apple Silicon runners. + +For more information: https://github.com/DeterminateSystems/nix-src/issues/224 + +PR: [DeterminateSystems/nix-src#257](https://github.com/DeterminateSystems/nix-src/pull/257) + +### Bugs fixed + +* IPv6 Store URLs now handles zone ID references like it did in previous releases [NixOS/nix#14434](https://github.com/NixOS/nix/pull/14434) + + +**Full Changelog**: [v3.12.2...v3.13.0](https://github.com/DeterminateSystems/nix-src/compare/v3.12.2...v3.13.0) diff --git a/doc/manual/source/release-notes-determinate/v3.13.1.md b/doc/manual/source/release-notes-determinate/v3.13.1.md new file mode 100644 index 000000000000..025a192c44ee --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.13.1.md @@ -0,0 +1,10 @@ +# Release 3.13.1 (2025-11-12) + +* Based on [upstream Nix 2.32.4](../release-notes/rl-2.32.md). + +## What's Changed +* nix bundle: Wait for async path writer by @edolstra in [DeterminateSystems/nix-src#260](https://github.com/DeterminateSystems/nix-src/pull/260) +* Sync with upstream 2.32.4 by @edolstra in [DeterminateSystems/nix-src#261](https://github.com/DeterminateSystems/nix-src/pull/261) + + +**Full Changelog**: [v3.13.0...v3.13.1](https://github.com/DeterminateSystems/nix-src/compare/v3.13.0...v3.13.1) diff --git a/doc/manual/source/release-notes-determinate/v3.13.2.md b/doc/manual/source/release-notes-determinate/v3.13.2.md new file mode 100644 index 000000000000..2490b865e6bc --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.13.2.md @@ -0,0 +1,68 @@ +# Release 3.13.2 (2025-11-19) + +* Based on [upstream Nix 2.32.4](../release-notes/rl-2.32.md). + +## What's Changed + +### Abbreviate flakerefs in lockfile diffs and `nix flake metadata` + +Flake refs are now abbreviated when possible, to reduce visual clutter. + +For example, this changes + +``` +• Updated input 'blender-bin': + 'https://api.flakehub.com/f/pinned/edolstra/blender-bin/1.0.19/01993ca7-2aa8-746f-96f5-ca8d2c2b962d/source.tar.gz?narHash=sha256-ZqVhVl9UYVErF8HW8lcvqss005VWYjuX//rZ%2BOmXyHg%3D' (2025-09-12) + → 'https://api.flakehub.com/f/pinned/edolstra/blender-bin/1.0.20/019a8772-b044-7738-8c03-109bdc9f0a01/source.tar.gz?narHash=sha256-sVj9Gmx0kwTDQPJ5kgQYszE3Hdjevu0zx0b/bL2fyUc%3D' (2025-11-15) +• Updated input 'nix': + 'github:DeterminateSystems/nix-src/236ebef6514f3a2a9765c8a1d80dd503b8e672be?narHash=sha256-s6/Err0yqOp5fM3OdCF1vhmEYpeElbPOWX88YrW2qj4%3D' (2025-10-23) + → 'github:DeterminateSystems/nix-src/ef054dc06e9701597bce0b0572af18cb4c7e7277?narHash=sha256-uqYmH0KA8caQqX5u4BMarZsuDlC%2B71HRsH3h4f3DPCA%3D' (2025-11-12) +``` + +to + +``` +• Updated input 'blender-bin': + 'https://api.flakehub.com/f/pinned/edolstra/blender-bin/1.0.19/01993ca7-2aa8-746f-96f5-ca8d2c2b962d/source.tar.gz' (2025-09-12) + → 'https://api.flakehub.com/f/pinned/edolstra/blender-bin/1.0.20/019a8772-b044-7738-8c03-109bdc9f0a01/source.tar.gz' (2025-11-15) +• Updated input 'nix': + 'github:DeterminateSystems/nix-src/236ebef' (2025-10-23) + → 'github:DeterminateSystems/nix-src/ef054dc' (2025-11-12) +``` + +PR: [DeterminateSystems/nix-src#264](https://github.com/DeterminateSystems/nix-src/pull/264) + +### `nix flake prefetch-inputs` now skips build-time inputs + +Build-time inputs can already be fetched in parallel, so prefetching them is usually not what you want. + +This can be especially noticeable in projects that make extensive use of build-time flake inputs. + +PR: [DeterminateSystems/nix-src#263](https://github.com/DeterminateSystems/nix-src/pull/263) + +### Don't compute `revCount`/`lastModified` if they're already specified + +We don't care if the user (or more likely the lock file) specifies an incorrect value for these attributes, since it doesn't matter for security (unlike content hashes like `narHash`). + +This can save time when operating on large repos -- having to recalculate these attributes could slow things down greatly. + +PR: [DeterminateSystems/nix-src#269](https://github.com/DeterminateSystems/nix-src/pull/269) + +### Avoid unnecessary Git refetches + +This fixes the issue where updating a Git input does a non-shallow fetch, and then a subsequent eval does a shallow refetch because the `revCount` is already known. + +Now the subsequent eval will reuse the repo used in the first fetch. + +PR: [DeterminateSystems/nix-src#270](https://github.com/DeterminateSystems/nix-src/pull/270) + +### Use our mirrored flake registry + +The flake registry is security-critical and thus should have high availability. + +By mirroring the upstream Nix flake registry, we can make it less likely that a GitHub outage affects being able to resolve from the registry. + +PR: [DeterminateSystems/nix-src#271](https://github.com/DeterminateSystems/nix-src/pull/271) + + +**Full Changelog**: [v3.13.1...v3.13.2](https://github.com/DeterminateSystems/nix-src/compare/v3.13.1...v3.13.2) diff --git a/doc/manual/source/release-notes-determinate/v3.14.0.md b/doc/manual/source/release-notes-determinate/v3.14.0.md new file mode 100644 index 000000000000..d72d5d21468c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.14.0.md @@ -0,0 +1,159 @@ +# Release 3.14.0 (2025-12-08) + +* Based on [upstream Nix 2.32.4](../release-notes/rl-2.32.md). + +## What is going on?! `nix ps` to the rescue + +Determinate Nix now features a `nix ps` command to summarize all of the active builds and child processes: + +``` +$ nix ps +USER PID CPU DERIVATION/COMMAND +_nixbld1 30167 0.4s /nix/store/h431bcfml83czhpyzljhp9mw4yrq95vs-determinate-nix-manual-3.14.0.drv (wall=9s) +_nixbld1 30167 0.2s └───bash -e /nix/store/jwqf79v5p51x9mv8vx20fv9mzm2x7kig-source-stdenv.sh /nix/store/285whzixr5k1kfj6nidyj29mqqgv7n0b-default-builder.s +_nixbld1 30278 0.0s └───ninja -j14 +_nixbld1 30279 0.0s ├───/nix/store/p7rag2cw99d7alp6749rjqp71qc0mnzl-python3-3.12.11/bin/python3.12 /nix/store/8k5fancbc5fjmxq6izn0z4inwnmpj09y-mes +_nixbld1 30286 0.0s │ └───/nix/store/z59zm01pjwzil2qkvv0s4ibk54risy9a-determinate-nix-3.14.0/bin/nix config show --json +_nixbld1 30280 0.0s ├───/nix/store/p7rag2cw99d7alp6749rjqp71qc0mnzl-python3-3.12.11/bin/python3.12 /nix/store/8k5fancbc5fjmxq6izn0z4inwnmpj09y-mes +_nixbld1 30287 0.0s │ └───/nix/store/z59zm01pjwzil2qkvv0s4ibk54risy9a-determinate-nix-3.14.0/bin/nix __dump-language +_nixbld1 30281 0.0s ├───/nix/store/p7rag2cw99d7alp6749rjqp71qc0mnzl-python3-3.12.11/bin/python3.12 /nix/store/8k5fancbc5fjmxq6izn0z4inwnmpj09y-mes +_nixbld1 30288 0.0s │ └───/nix/store/z59zm01pjwzil2qkvv0s4ibk54risy9a-determinate-nix-3.14.0/bin/nix __dump-cli +_nixbld1 30282 0.0s ├───/nix/store/p7rag2cw99d7alp6749rjqp71qc0mnzl-python3-3.12.11/bin/python3.12 /nix/store/8k5fancbc5fjmxq6izn0z4inwnmpj09y-mes +_nixbld1 30284 0.0s │ └───/nix/store/z59zm01pjwzil2qkvv0s4ibk54risy9a-determinate-nix-3.14.0/bin/nix __dump-xp-features +_nixbld1 30283 0.0s └───/nix/store/p7rag2cw99d7alp6749rjqp71qc0mnzl-python3-3.12.11/bin/python3.12 /nix/store/8k5fancbc5fjmxq6izn0z4inwnmpj09y-mes +_nixbld1 30285 0.0s └───/nix/store/bs1pvy8margy5sj0jwahchxbjnqzi14i-bash-5.2p37/bin/bash -euo pipefail -c if type -p build-release-notes > /de +_nixbld1 30289 0.0s └───changelog-d ../source/release-notes/../../rl-next +``` + +For the integrators out there, it also has a `--json` flag with all the raw data. + +PRs: +* [DeterminateSystems/nix-src#282](https://github.com/DeterminateSystems/nix-src/pull/282) +* [DeterminateSystems/nix-src#287](https://github.com/DeterminateSystems/nix-src/pull/287) + + +## Nix `build`, `profile`, and `flake check` commands tell you what output failed + +These commands now tell you exactly what flake outputs failed to build. +Previously, the error would indicate only what derivation failed to build -- but not which output. + +Now, `nix build` and `nix profile` commands provide the specific output: + +``` +$ nix build .#oneFakeHash .#badSystem --keep-going +❌ git+file:///Users/grahamc/src/github.com/DeterminateSystems/samples#oneFakeHash +error: hash mismatch in fixed-output derivation '/nix/store/58pp1y74j4f5zxfq50xncv2wvnxf7w3y-one-fake-hash.drv': + specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + got: sha256-i7j83d71sibS/ssSjLJ5PMKmbhjAM+BHW0aElvkgEwY= +❌ git+file:///Users/grahamc/src/github.com/DeterminateSystems/samples#badSystem +error: Cannot build '/nix/store/5vsaxi730yl2icngkyvn8wiflik5wfmq-bad-system.drv'. + Reason: required system or feature not available + Required system: 'bogus' with features {} + Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} +``` + +And in a great change for CI, `nix flake check` users get improved summaries too: + +``` +$ nix flake check +❓ checks.aarch64-darwin.twoFakeHashes (cancelled) +❓ checks.aarch64-darwin.badSystemNested (cancelled) +❓ checks.aarch64-darwin.oneFakeHash (cancelled) +❓ checks.aarch64-darwin.failure (cancelled) +❓ checks.aarch64-darwin.badSystem (cancelled) +❓ checks.aarch64-darwin.weirdHash (cancelled) +❓ checks.aarch64-darwin.all (cancelled) +❓ checks.aarch64-darwin.fakeHashes (cancelled) +❓ checks.aarch64-darwin.incorrectHashes (cancelled) +❓ checks.aarch64-darwin.badFeaturesNested (cancelled) +❓ checks.aarch64-darwin.failureNested (cancelled) +❌ checks.aarch64-darwin.badFeatures +error: Cannot build '/nix/store/sc1cyhrpsm9yjx55cl2zzyr5lypwigi6-bad-feature.drv'. + Reason: required system or feature not available + Required system: 'aarch64-darwin' with features {bogus} + Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} +``` + +PRs: +* [DeterminateSystems/nix-src#281](https://github.com/DeterminateSystems/nix-src/pull/281) +* [DeterminateSystems/nix-src#285](https://github.com/DeterminateSystems/nix-src/pull/285) + + +## More seamless upgrades from Nix 2.18 and Nix 2.19 + +We've heard from some users who are trying to upgrade from Nix 2.18. + +These users are primarily experiencing problems caused by Nix 2.20 switching from `git-archive` to `libgit2` for fetching repositories. +This change caused some `git-archive` filters to stop executing, like autocrlf. +Not running those filters is an improvement, and running those filters *can cause* instability in source hashes. +However, this switch *did* cause previously valid hashes to become invalid. + +Determinate Nix now retries fetching an old archive with `git-archive` as a fallback when libgit2 fails to provide the correct source. + +Further, to support a progressive migration Determinate Nix has a new option: `nix-219-compat`. +Set `nix-219-compat=true` to cause Nix to author new flake.nix files with a `git-archive` based source hash. + +Finally, a user identified `builtins.path` changed since 2.18 and stopped propagating references. +We have corrected this regression. + +PRs: +* [DeterminateSystems/nix-src#283](https://github.com/DeterminateSystems/nix-src/pull/283) +* [DeterminateSystems/nix-src#278](https://github.com/DeterminateSystems/nix-src/pull/278) + +## Flake registry mirroring + +Determinate Nix now includes a fallback copy of the Nix Registry. +This change builds on top of v3.13.2, where we changed from the upstream Nix registry to a mirrored copy hosted by `install.determinate.systems`. + +Combined, these changes increase the reliability of Nix in the face of network outages. + +> [!NOTE] +> Flake registry URLs for `flake.nix` inputs is deprecated. +> The flake registry should only be used for interactive use. +> See: https://github.com/DeterminateSystems/nix-src/issues/37 + +PR: [DeterminateSystems/nix-src#273](https://github.com/DeterminateSystems/nix-src/pull/273) + +## Flake registry resolution CLI + +We added the new command `nix registry resolve` to help debug issues with Flake registries. +This command looks up a flake registry input name and returns the flakeref it resolves to. + +For example, looking up Nixpkgs: + +``` +$ nix registry resolve nixpkgs +github:NixOS/nixpkgs/nixpkgs-unstable +``` + +Or looking up the 25.11 branch of Nixpkgs: +``` +$ nix registry resolve nixpkgs/release-25.11 +github:NixOS/nixpkgs/release-25.11 +``` + +> [!NOTE] +> Flake registry URLs for `flake.nix` inputs is deprecated. +> The flake registry should only be used for interactive use. +> See: https://github.com/DeterminateSystems/nix-src/issues/37 + +PR: [DeterminateSystems/nix-src#273](https://github.com/DeterminateSystems/nix-src/pull/273) + +## Improved Docker image packaging + +Thanks to `employee-64c7dcd530593118dcccc3fb`, the OCI / Docker images built by the Determinate Nix flake.nix can be further customized. + +Users can specify their own base image by specifying `fromImage`. + +Additionally, users can specify additional directories to include at the beginning or end of the PATH variable with `extraPrePaths` and `extraPostPaths`. + +PRs: +* [DeterminateSystems/nix-src#277](https://github.com/DeterminateSystems/nix-src/pull/277) +* [DeterminateSystems/nix-src#280](https://github.com/DeterminateSystems/nix-src/pull/280) + +## Bug fixes + +* Corrected an error with parallel evaluation which ([DeterminateSystems/nix-src#286](https://github.com/DeterminateSystems/nix-src/pull/286)) +* Fixed compatibility with updated Nixpkgs versions. Thank you SandaruKasa! ([DeterminateSystems/nix-src#284](https://github.com/DeterminateSystems/nix-src/pull/284)) + +**Full Changelog**: [v3.13.2...v3.14.0](https://github.com/DeterminateSystems/nix-src/compare/v3.13.2...v3.14.0) diff --git a/doc/manual/source/release-notes-determinate/v3.15.0.md b/doc/manual/source/release-notes-determinate/v3.15.0.md new file mode 100644 index 000000000000..fb568374c3f2 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.15.0.md @@ -0,0 +1,28 @@ +# Release 3.15.0 (2025-12-19) + +* Based on [upstream Nix 2.33.0](../release-notes/rl-2.33.md). + +## `fetchTree` improvement + +`builtins.fetchTree` now implicitly treats the fetched tree as "final" when a `narHash` is supplied, meaning that it will not return attributes like `lastModified` or `revCount` unless they were specified by the caller. This makes it possible to substitute the tree from a binary cache, which is often more efficient. Furthermore, for Git inputs, it allows Nix to perform a shallow fetch, which is much faster. + +This is primarily useful for users of `flake-compat`, since it uses `builtins.fetchTree` internally. + +PR: [DeterminateSystems/nix-src#297](https://github.com/DeterminateSystems/nix-src/pull/297) + +## New builtin function `builtins.filterAttrs` + +Nixpkgs heavily relies on this function to select attributes from an attribute set: + +```nix +filterAttrs = pred: set: removeAttrs set (filter (name: !pred name set.${name}) (attrNames set)); +``` + +Determinate Nix now has this function built-in, which makes it much faster. + +PR: [DeterminateSystems/nix-src#291](https://github.com/DeterminateSystems/nix-src/pull/291) + +## New Contributors +* @not-ronjinger made their first contribution in [DeterminateSystems/nix-src#291](https://github.com/DeterminateSystems/nix-src/pull/291) + +**Full Changelog**: [v3.14.0...v3.15.0](https://github.com/DeterminateSystems/nix-src/compare/v3.14.0...v3.15.0) diff --git a/doc/manual/source/release-notes-determinate/v3.15.1.md b/doc/manual/source/release-notes-determinate/v3.15.1.md new file mode 100644 index 000000000000..9243962cf4b5 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.15.1.md @@ -0,0 +1,15 @@ +# Release 3.15.1 (2025-12-24) + +* Based on [upstream Nix 2.33.0](../release-notes/rl-2.33.md). + +## What's Changed +Users reported the v3.15.0 tarball could not be fetched in a fixed-output derivation due to current stdenv paths present in the documentation. This release eliminated those paths. + +PR: [DeterminateSystems/nix-src#306](https://github.com/DeterminateSystems/nix-src/pull/306) + +Additionally, this change re-enables CodeRabbit's code review on our changes. CodeRabit was disabled by the upstream project, and we inadvertently included that change. + +PR: [DeterminateSystems/nix-src#305](https://github.com/DeterminateSystems/nix-src/pull/305) + + +**Full Changelog**: [v3.15.0...v3.15.1](https://github.com/DeterminateSystems/nix-src/compare/v3.15.0...v3.15.1) diff --git a/doc/manual/source/release-notes-determinate/v3.15.2.md b/doc/manual/source/release-notes-determinate/v3.15.2.md new file mode 100644 index 000000000000..c5e5339990b5 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.15.2.md @@ -0,0 +1,44 @@ +# Release 3.15.2 (2026-01-20) + +* Based on [upstream Nix 2.33.1](../release-notes/rl-2.33.md). + +## What's Changed + +### Improved performance for users with a lot of dependencies + +If you even had the occasion to query your binary cache for over 110,000 store path simultaneously you might have found it rather slow. +Previously, Nix would enqueue all the downloads at once. +This appears to trigger quadratic behavior in curl. + +Determinate Nix now enqueues a reasonable number of subtitutions once. +At the same time, we fixed a performance issue in the progress bar with so many dependencies. + +PR: [DeterminateSystems/nix-src#315](https://github.com/DeterminateSystems/nix-src/pull/315) + +### Lazy trees update: path inputs are now lazy + +Previously inputs like `path:///path/to/a/dependency` were eagerly fetched when lazy-trees is enabled. +In Determinate Nix 3.15.2, path input types are also fetched lazily. +This change saves time and improves performance for users with path inputs. + +PRs: +* [DeterminateSystems/nix-src#312](https://github.com/DeterminateSystems/nix-src/pull/312) +* [DeterminateSystems/nix-src#317](https://github.com/DeterminateSystems/nix-src/pull/317) + +### `nix repl` now reports the Determinate version + +A small change, but now `nix repl` correctly reports the Determinate Nix version: + +``` +$ nix repl +Nix (Determinate Nix 3.15.1) 2.33.0 +Type :? for help. +nix-repl> +``` + +PR: [DeterminateSystems/nix-src#316](https://github.com/DeterminateSystems/nix-src/pull/316) + +## New Contributors +* @dliberalesso made their first contribution in [DeterminateSystems/nix-src#313](https://github.com/DeterminateSystems/nix-src/pull/313) + +**Full Changelog**: [v3.15.1...v3.15.2](https://github.com/DeterminateSystems/nix-src/compare/v3.15.1...v3.15.2) diff --git a/doc/manual/source/release-notes-determinate/v3.16.0.md b/doc/manual/source/release-notes-determinate/v3.16.0.md new file mode 100644 index 000000000000..8e80ac68402a --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.16.0.md @@ -0,0 +1,53 @@ +# Release 3.16.0 (2026-02-12) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## Support `.gitattributes` in subdirectories + +For performance, the Git backwards compatibility hack was only applied to repositories that had a `.gitattributes` in the root directory. +However, it is possible to have a `.gitattributes` file in a subdirectory, and there are real-world repos that do this, so we have dropped that restriction. + +PR: [DeterminateSystems/nix-src#335](https://github.com/DeterminateSystems/nix-src/pull/335) + +## Fix hung downloads when `http-connections = 0` + +When we started limiting the number of active cURL handles in [DeterminateSystems/nix-src#315](https://github.com/DeterminateSystems/nix-src/pull/315), we did not take into account that `http-connections = 0` is a special value that means, roughly "as many connections as possible" (the exact behavior is up to cURL). + +This should now be fixed. + +PR: [DeterminateSystems/nix-src#327](https://github.com/DeterminateSystems/nix-src/pull/327) + +## `builtins.getFlake` now supports relative paths + +`builtins.getFlake` now supports using relative paths, like: + +```nix +builtins.getFlake ./.. +``` + +instead of the hacky + +```nix +builtins.getFlake (builtins.flakeRefToString { type = "path"; path = self.sourceInfo.outPath; narHash = self.narHash; }); +``` + +Note that allowing `builtins.getFlake` to fetch from store paths is probably a bad idea, since it's ambiguous when using chroot stores, so a warning will be printed when this is encountered. + +PRs: +* [DeterminateSystems/nix-src#337](https://github.com/DeterminateSystems/nix-src/pull/337) +* [DeterminateSystems/nix-src#338](https://github.com/DeterminateSystems/nix-src/pull/338) + +## Fixed a bug with too many open files + +Recently, some users have reported seeing errors like: + +``` +error: creating git packfile indexer: failed to create temporary file '/Users/anon/.cache/nix/tarball-cache-v2/objects/pack/pack_git2_56d617039ac17c2b': Too many open files +``` + +This should now be fixed. + +PR: [DeterminateSystems/nix-src#347](https://github.com/DeterminateSystems/nix-src/pull/347) + + +**Full Changelog**: [v3.15.2...v3.16.0](https://github.com/DeterminateSystems/nix-src/compare/v3.15.2...v3.16.0) diff --git a/doc/manual/source/release-notes-determinate/v3.16.1.md b/doc/manual/source/release-notes-determinate/v3.16.1.md new file mode 100644 index 000000000000..6ecd5262b7c9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.16.1.md @@ -0,0 +1,24 @@ +# Release 3.16.1 (2026-02-22) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed + +### `nix store info` now correctly support `--refresh` and `--offline` + +Previously, Nix had a hard-coded TTL of seven days. +Determinate Nix moved that TTL to a new setting `narinfo-cache-meta-ttl` and now `nix store info` respects the `--refresh` and `--offline` flags. + +This change makes it possible to freshly validate authenticating to a remote store. + +PR: [DeterminateSystems/nix-src#355](https://github.com/DeterminateSystems/nix-src/pull/355) + +### Corrected `builtins.hashString` behavior under lazy trees + +`builtins.hashString` now devirtualizes lazy paths, making the hash result stable. + +PR: [DeterminateSystems/nix-src#360](https://github.com/DeterminateSystems/nix-src/pull/360) + + + +**Full Changelog**: [v3.16.0...v3.16.1](https://github.com/DeterminateSystems/nix-src/compare/v3.16.0...v3.16.1) diff --git a/doc/manual/source/release-notes-determinate/v3.16.2.md b/doc/manual/source/release-notes-determinate/v3.16.2.md new file mode 100644 index 000000000000..73a1b25f21c8 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.16.2.md @@ -0,0 +1,8 @@ +# Release 3.16.2 (2026-02-23) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed +This release is exclusively improvements to `determinate-nixd`. + + diff --git a/doc/manual/source/release-notes-determinate/v3.16.3.md b/doc/manual/source/release-notes-determinate/v3.16.3.md new file mode 100644 index 000000000000..fcc6fefa33c7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.16.3.md @@ -0,0 +1,6 @@ +# Release 3.16.3 (2026-02-24) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed +This release only includes changes in determinate-nixd. diff --git a/doc/manual/source/release-notes-determinate/v3.17.0.md b/doc/manual/source/release-notes-determinate/v3.17.0.md new file mode 100644 index 000000000000..e09938786e55 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.17.0.md @@ -0,0 +1,10 @@ +# Release 3.17.0 (2026-03-04) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed +Determinate Nix 3.17.0 brings exciting improvements like Flake Schemas, provenance, and Wasm / WASI. +We'll be posting more details over the next week or so on our blog: https://determinate.systems/blog/. + + +**Full Changelog**: [v3.16.3...v3.17.0](https://github.com/DeterminateSystems/nix-src/compare/v3.16.3...v3.17.0) diff --git a/doc/manual/source/release-notes-determinate/v3.17.1.md b/doc/manual/source/release-notes-determinate/v3.17.1.md new file mode 100644 index 000000000000..19da0209931e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.17.1.md @@ -0,0 +1,49 @@ +# Release 3.17.1 (2026-03-18) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed + +### Provenance now supports "build-time" provenance tags + +Users can now specify key/value tags to attach to a build's provenance. +For example, the `nix-installer-action` now attaches the following properties to each build: + +- github_workflow_ref +- github_workflow_sha +- github_sha +- github_run_attempt +- github_run_id +- github_run_number +- github_job +- github_ref +- github_repository +- github_server_url + +PR: [DeterminateSystems/nix-src#374](https://github.com/DeterminateSystems/nix-src/pull/374) + +### Flake inputs are substituted when possible + +Locked flake inputs will be fetched from a binary cache when possible, instead of preferring the authoritative flake source. +This is intended to reduce load on code forges, and also improves the user experience on large flake inputs. + +PR: [DeterminateSystems/nix-src#380](https://github.com/DeterminateSystems/nix- + +### `nix profile upgrade` and `nix profile remove` now support tab completion + +PR: [DeterminateSystems/nix-src#382](https://github.com/DeterminateSystems/nix-src/pull/382) + +### Flake schemas can now define an output as "legacy" + +"Legacy" flakes are intended for legacyPackages on Nixpkgs. +The "legacy" mark is intended to reduce evaluation time due to the extreme size of legacyPackages. +Note: the name "legacy" is not intended as a value judgement, and at this point we're sort of stuck with the name. + +### Bug fixes + +* Fix crash in `nix repl` loading an invalid WASM file twice. [DeterminateSystems/nix-src#378](https://github.com/DeterminateSystems/nix-src/pull/378) +* Don't crash if SIGINT happens while printing an exception. [DeterminateSystems/nix-src#384](https://github.com/DeterminateSystems/nix-src/pull/384) +* `nix-env -i`: Wait for the async path writer. [DeterminateSystems/nix-src#385](https://github.com/DeterminateSystems/nix-src/pull/385) + + +**Full Changelog**: [v3.17.0...v3.17.1](https://github.com/DeterminateSystems/nix-src/compare/v3.17.0...v3.17.1) diff --git a/doc/manual/source/release-notes-determinate/v3.17.2.md b/doc/manual/source/release-notes-determinate/v3.17.2.md new file mode 100644 index 000000000000..ce45a8df7ec7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.17.2.md @@ -0,0 +1,16 @@ +# Release 3.17.2 (2026-03-27) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed + +### Bug fixes + +* Cache `getLegacyGitAccessor()`. [DeterminateSystems/nix-src#396](https://github.com/DeterminateSystems/nix-src/pull/396) +* Don't destroy `windowSize` mutex. [DeterminateSystems/nix-src#397](https://github.com/DeterminateSystems/nix-src/pull/397) +* When doing concurrent substitutions of the same path, download only once. [DeterminateSystems/nix-src#398](https://github.com/DeterminateSystems/nix-src/pull/398) +* `builtins.getFlake`: Handle `path:

` where *p* has a discarded string context. [DeterminateSystems/nix-src#402](https://github.com/DeterminateSystems/nix-src/pull/402) +* Ensure `_interruptCallbacks` is alive while `signalHandlerThread` is running. [DeterminateSystems/nix-src#403](https://github.com/DeterminateSystems/nix-src/pull/403) +* Fix assertion failure in `nix::BuiltPath::toRealisedPaths()`. [DeterminateSystems/nix-src#401](https://github.com/DeterminateSystems/nix-src/pull/401) + +**Full Changelog**: [v3.17.1...v3.17.2](https://github.com/DeterminateSystems/nix-src/compare/v3.17.1...v3.17.2) diff --git a/doc/manual/source/release-notes-determinate/v3.17.3.md b/doc/manual/source/release-notes-determinate/v3.17.3.md new file mode 100644 index 000000000000..24463e48485f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.17.3.md @@ -0,0 +1,48 @@ +# Release 3.17.3 (2026-04-07) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed + +### Fix for GHSA-g3g9-5vj6-r3gj: root privilege escalation via the Nix daemon + +This release contains a critical fix for a security vulnerability on Linux that allows any user that has access to the Nix daemon to obtain root privileges. +This vulnerability affects all versions of Determinate Nix prior to 3.17.3, and all versions of upstream Nix prior to 2.34.5, 2.33.4, 2.32.7, 2.31.4, 2.30.4, 2.29.3, and 2.28.6. +All Linux users are advised to upgrade immediately. + +For more details, see the upstream advisory [GHSA-g3g9-5vj6-r3gj](https://github.com/NixOS/nix/security/advisories/GHSA-g3g9-5vj6-r3gj). +Many thanks to edef for reporting this issue and to Sergei Zimmerman for implementing the fix. + +### Avoiding duplicate source tree downloads + +When multiple Nix processes (like `nix-eval-jobs` instances) fetch the same source tree at the same time, it was previously possible for each process to perform the download independently, resulting in wasteful multiple downloads of the same source tree. Nix now uses a per-source tree lock to ensure that only one process performs the download. + +PR: [DeterminateSystems/nix-src#410](https://github.com/DeterminateSystems/nix-src/pull/410) + +### WAT support in `builtins.wasm` + +`builtins.wasm` now supports WebAssembly Text Format (WAT) in addition to binary Wasm modules. +This is primarily useful for testing. + +PR: [DeterminateSystems/nix-src#405](https://github.com/DeterminateSystems/nix-src/pull/405) + +### Git shallow fetching + +Nix now removes Git's `shallow.lock` lock file before running `git fetch`. +This prevents fetches from hanging if Git was previously interrupted. + +PR: [DeterminateSystems/nix-src#414](https://github.com/DeterminateSystems/nix-src/pull/414) + +### Debugging improvements + +Certain C++ exceptions that should never happen (like `std::logic_error`) are now treated as aborts, providing stack traces and core dumps that are easier to debug. + +PR: [DeterminateSystems/nix-src#407](https://github.com/DeterminateSystems/nix-src/pull/407) + +### Bug fixes + +* `nix develop` respects `legacyPackages` again. [DeterminateSystems/nix-src#413](https://github.com/DeterminateSystems/nix-src/pull/413) + +* The `lookupPathResolved` data structure has been made thread-safe. [DeterminateSystems/nix-src#415](https://github.com/DeterminateSystems/nix-src/pull/415) + +**Full Changelog**: [v3.17.2...v3.17.3](https://github.com/DeterminateSystems/nix-src/compare/v3.17.2...v3.17.3) diff --git a/doc/manual/source/release-notes-determinate/v3.18.0.md b/doc/manual/source/release-notes-determinate/v3.18.0.md new file mode 100644 index 000000000000..aaf22deec8c8 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.18.0.md @@ -0,0 +1,42 @@ +# Release 3.18.0 (2026-04-20) + +* Based on [upstream Nix 2.33.3](../release-notes/rl-2.33.md). + +## What's Changed + +### Sentry integration + +In order to more proactively keep track of crashes, Sentry is now integrated into Determinate Nix. + +This allows us to more easily triage and remedy crashes that occur in the wild, without depending on manual use reports. + +It can be enabled by: + +* populating the file `/etc/nix/sentry-endpoint` with a Sentry DSN; or +* setting the `NIX_SENTRY_ENDPOINT` environment variable to a Sentry DSN + +and can be disabled by: + +* setting the environment variable `DETSYS_IDS_TELEMETRY` to the value `disabled`; or +* setting the environment variable `NIX_SENTRY_ENDPOINT` to an empty string + +PR: [DeterminateSystems/nix-src#418](https://github.com/DeterminateSystems/nix-src/pull/418) + +### Pre-build hook now receives the JSON serialization of the derivation + +The pre-build hook already received the path of the derivation as an argument, but that path doesn't typically exist when called as a remote build. + +Now, the pre-build hook is spawned with the environment variable `NIX_DERIVATION_V4` set to a file that contains the JSON representation of the derivation in v4 format, allowing instrospection of e.g. `requiredSystemFeatures` for scheduling decisions. + +PR: [DeterminateSystems/nix-src#424](https://github.com/DeterminateSystems/nix-src/pull/424) + +### Fix empty Git exports when using legacy Git compatibility + +A regression introduced in v3.16.0 made it possible for there to be empty Git exports in the Nix store when using legacy Git flakes (those depending on Nix 2.19 lockfile semantics). + +This is now fixed. + +PR: [DeterminateSystems/nix-src#425](https://github.com/DeterminateSystems/nix-src/pull/425) + + +**Full Changelog**: [v3.17.3...v3.18.0](https://github.com/DeterminateSystems/nix-src/compare/v3.17.3...v3.18.0) diff --git a/doc/manual/source/release-notes-determinate/v3.18.1.md b/doc/manual/source/release-notes-determinate/v3.18.1.md new file mode 100644 index 000000000000..26c9592561ad --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.18.1.md @@ -0,0 +1,23 @@ +# Release 3.18.1 (2026-04-23) + +* Based on [upstream Nix 2.33.4](../release-notes/rl-2.33.md). + +## What's Changed + +### Sentry integration improvements + +This release includes fixes for a couple of issues with the Sentry integration: + +* Ensures the chroot helper starts before the Sentry thread, allowing chroot stores to work again +* Resets Mach exception ports on macOS, so that exec'd programs do not communicate with Determinate Nix's `crashpad_handler` + +Additionally, Determinate Nix now includes the Nix command and subcommand to Sentry reports to make it easier to discern where an issue happened. +Not that this does _not_ include any command-line arguments other than the command (such as `nix-daemon` and `nix`) and subcommand (such as `flake show`). + +PRs: +* [DeterminateSystems/nix-src#433](https://github.com/DeterminateSystems/nix-src/pull/433) +* [DeterminateSystems/nix-src#432](https://github.com/DeterminateSystems/nix-src/pull/432) +* [DeterminateSystems/nix-src#436](https://github.com/DeterminateSystems/nix-src/pull/436) + + +**Full Changelog**: [v3.18.0...v3.18.1](https://github.com/DeterminateSystems/nix-src/compare/v3.18.0...v3.18.1) diff --git a/doc/manual/source/release-notes-determinate/v3.19.0.md b/doc/manual/source/release-notes-determinate/v3.19.0.md new file mode 100644 index 000000000000..9b0713fd3bbe --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.19.0.md @@ -0,0 +1,20 @@ +# Release 3.19.0 (2026-04-28) + +* Based on [upstream Nix 2.34.6](../release-notes/rl-2.34.md). + +## What's Changed + +### Update to upstream Nix 2.34 + +Determinate Nix is now based on [upstream Nix 2.34](../release-notes/rl-2.34.md), which includes a large number of bug fixes and improvements. + +### Provenance is preserved when copying paths to the daemon + +Previously, `nix copy` would not forward provenance information to the recipient. + +This has been fixed, such that building a derivation on machine A and `nix copy`ing it to machine B will show the same provenance. + +PR: [DeterminateSystems/nix-src#399](https://github.com/DeterminateSystems/nix-src/pull/399) + + +**Full Changelog**: [v3.18.1...v3.19.0](https://github.com/DeterminateSystems/nix-src/compare/v3.18.1...v3.19.0) diff --git a/doc/manual/source/release-notes-determinate/v3.19.1.md b/doc/manual/source/release-notes-determinate/v3.19.1.md new file mode 100644 index 000000000000..dcbd608b243c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.19.1.md @@ -0,0 +1,35 @@ +# Release 3.19.1 (2026-05-04) + +* Based on [upstream Nix 2.34.6](../release-notes/rl-2.34.md). + +## What's Changed + +### Fix for GHSA-vh5x-56v6-4368: Coroutine stack-to-heap overflow via unbounded recursion in NAR directory parser + +Unbounded recursion in the NAR (Nix Archive) parser could lead to a stack-to-heap overflow when the parser is run on a coroutine stack. The stack was allocated without a guard page, which meant that a stack overflow could overwrite memory on the heap and could allow for arbitrary code execution as the Nix daemon (run as root in multi-user installations) if ASLR hardening is bypassed. This vulnerability can be exploited by all users able to connect to the Nix daemon (configurable via allowed-users setting - all users by default). + +The vulnerability is present since 2.24.4. + +For more details, see the upstream advisory [GHSA-vh5x-56v6-4368](https://github.com/NixOS/nix/security/advisories/GHSA-vh5x-56v6-4368). + +Many thanks to edef for reporting this issue and to Sergei Zimmerman and edef for implementing the fix and several hardening improvements. + +### Fix for GHSA-gr92-w2r5-qw5p: Absolute path traversal when unpacking archives to disk + +When an archive (e.g. a tarball) contains entries with absolute paths, `nix-prefetch-url --unpack`, `nix store prefetch-file --unpack` commands would write to a location outside the extraction root. This also affects `builtin:unpack-channel` builtin derivation builder, but it's sandboxed the same way as regular derivations builds, so in sandboxed builds it can't be used to achieve an arbitrary file write on the host filesystem. + +Other archive unpacking (`builtins.fetchTarball`, `builtins.fetchTree` and tarball-based flake inputs) is not affected, since Nix stores the unpacked tarball in a bare git repository (`.cache/nix/tarball-cache` or `.cache/nix/tarball-cache-v2`). + +This primarily affects packagers running `nix-prefetch-url --unpack` on untrusted archives. + +The vulnerability is present since 2.24.7. + +For more details, see the upstream advisory [GHSA-gr92-w2r5-qw5p](https://github.com/NixOS/nix/security/advisories/GHSA-gr92-w2r5-qw5p). + +Many thanks to edef for reporting this issue and to Sergei Zimmerman for implementing the fix. + +### Bug fixes + +* A fix for random crashes on exit in the `curlFileTransfer` destructor. [DeterminateSystems/nix-src#444](https://github.com/DeterminateSystems/nix-src/pull/444) + +**Full Changelog**: [v3.19.0...v3.19.1](https://github.com/DeterminateSystems/nix-src/compare/v3.19.0...v3.19.1) diff --git a/doc/manual/source/release-notes-determinate/v3.20.0.md b/doc/manual/source/release-notes-determinate/v3.20.0.md new file mode 100644 index 000000000000..b077f99dde1d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.20.0.md @@ -0,0 +1,22 @@ +# Release 3.20.0 (2026-05-07) + +* Based on [upstream Nix 2.34.6](../release-notes/rl-2.34.md). + +## Support resuming from binary caches that don't support ranged requests + +Previously, Nix would just fail a download if something happened in the TCP or TLS layer, meaning a flaky proxy could fail to download a store path and fall back to building, when a retry might have not been flaky. + +Now, Determinate Nix will retry these failed substitutions, so you don't have to waste time building something that was cached but some proxy decided to cut you off early. + +PR: [DeterminateSystems/nix-src#445](https://github.com/DeterminateSystems/nix-src/pull/445) + +## Add a session ID to JSON log messages + +Determinate Nix now includes a freeform string session ID (`sid`) in its JSON log messages, which allows a anything that consumes these log messages to see which messages belong to the same session. + +By default, every Nix command invocation is its own session, but you can use the `session-id` setting to give a group of invocations the same session (otherwise, a random UUIDv7 like `019e037b-f603-7443-8071-cfd8585ec8c6` is set as the session ID). + +PR: [DeterminateSystems/nix-src#440](https://github.com/DeterminateSystems/nix-src/pull/440) + + +**Full Changelog**: [v3.19.1...v3.20.0](https://github.com/DeterminateSystems/nix-src/compare/v3.19.1...v3.20.0) diff --git a/doc/manual/source/release-notes-determinate/v3.21.0.md b/doc/manual/source/release-notes-determinate/v3.21.0.md new file mode 100644 index 000000000000..f06448792e91 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.0.md @@ -0,0 +1,44 @@ +# Release 3.21.0 (2026-05-22) + +* Based on [upstream Nix 2.34.6](../release-notes/rl-2.34.md). + +## CNSA 1.0 and 2.0 signing algorithm support + +Determinate Nix now supports a subset of CNSA 1.0 and 2.0 signature algorithms: `ecdsa-p384`, `ml-dsa-44`, `ml-dsa-65`, and `ml-dsa-87`. ML-DSA is a post-quantum cryptography signature scheme. +Support for these algorithms is gated behind the experimental feature `cnsa`. + +To use these signatures, enable the `cnsa` experimental feature and call `nix key generate-secret` with `--key-type ecdsa-p384|ml-dsa-{44,65,87}`. +Otherwise, everything works the same as the existing `ed25519` (libsodium) signatures, except that they produce larger keys/signatures (especially ML-DSA). + +We've also added commands `nix key convert-{public,secret}-to-pem`, which may be useful if you want to use the keys with the `openssl` CLI. + +PR: [DeterminateSystems/nix-src#449](https://github.com/DeterminateSystems/nix-src/pull/449) + +## `nix serve` command + +The new `nix serve` command provides a built-in binary cache server, similar to [`nix-serve`](https://github.com/edolstra/nix-serve/). +It's mostly intended for testing and to serve as a reference implementation of binary caches. + +PR: [DeterminateSystems/nix-src#428](https://github.com/DeterminateSystems/nix-src/pull/428) + +## Lazy `fetchTarball` + +`fetchTarball` is now lazy, which can prevent unnecessary copying of source trees into the Nix store. + +PRs: +* [DeterminateSystems/nix-src#468](https://github.com/DeterminateSystems/nix-src/pull/468) +* [DeterminateSystems/nix-src#469](https://github.com/DeterminateSystems/nix-src/pull/469) + +## Various bug fixes + +* Restore resBuildResult fields for failed builds [DeterminateSystems/nix-src#453](https://github.com/DeterminateSystems/nix-src/pull/453) +* parseString(): Fix out-of-bounds read [DeterminateSystems/nix-src#454](https://github.com/DeterminateSystems/nix-src/pull/454) +* filetransfer: handle exceptions thrown from enqueueItem [DeterminateSystems/nix-src#348](https://github.com/DeterminateSystems/nix-src/pull/348) +* libfetchers: verify git-lfs returns the same objects as git [DeterminateSystems/nix-src#456](https://github.com/DeterminateSystems/nix-src/pull/456) +* HttpBinaryCacheStore: Don't ignore 401/407 errors [DeterminateSystems/nix-src#461](https://github.com/DeterminateSystems/nix-src/pull/461) +* Remove non-atomically initialised variable vImportedDrvToDerivation [DeterminateSystems/nix-src#464](https://github.com/DeterminateSystems/nix-src/pull/464) +* Re-enable TCP keep-alive and handle S3's XML errors [DeterminateSystems/nix-src#463](https://github.com/DeterminateSystems/nix-src/pull/463) +* Fix logging segfault by leaking logger [DeterminateSystems/nix-src#470](https://github.com/DeterminateSystems/nix-src/pull/470) + + +**Full Changelog**: [v3.20.0...v3.21.0](https://github.com/DeterminateSystems/nix-src/compare/v3.20.0...v3.21.0) diff --git a/doc/manual/source/release-notes-determinate/v3.21.1.md b/doc/manual/source/release-notes-determinate/v3.21.1.md new file mode 100644 index 000000000000..2e9cb30b929f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.1.md @@ -0,0 +1,58 @@ +# Release 3.21.1 (2026-06-08) + +* Based on [upstream Nix 2.34.7](../release-notes/rl-2.34.md). + +## Faster `nix copy` + +Previously, `nix copy` processed store paths in topological order, meaning that if store path A depends on B, then Nix wouldn't even start the fetch of store path A from the source store, or the writing of store path A to the destination store, until store path B had been added to the destination store. + +Now, Nix can start fetching and writing store paths before their dependencies have been added to the destination store. This can provide significant speedups. For instance, the command +```console +# nix copy \ + --from file:///tmp/binary-cache \ + --to file:///tmp/binary-cache-2?compression=xz \ + --no-check-sigs \ + /nix/store/8xyk1qxxjfb8cm62g61yc59phwha92w7-kdenlive-25.08.3 +``` +went from 117.7s to 47.2s on a machine with 24 cores. + +This optimisation affects copying to binary caches and local stores, but not when going through the Nix daemon. It also does not speed up substitution yet. + +PRs: [DeterminateSystems/nix-src#477](https://github.com/DeterminateSystems/nix-src/pull/477), [DeterminateSystems/nix-src#478](https://github.com/DeterminateSystems/nix-src/pull/478), [DeterminateSystems/nix-src#481](https://github.com/DeterminateSystems/nix-src/pull/481), [DeterminateSystems/nix-src#480](https://github.com/DeterminateSystems/nix-src/pull/480) + +## Determinate Secure Packages + +Determinate Nix is now built against [Determinate Secure Packages](https://determinate.systems/secure-packages/), a distribution of Nixpkgs that adds enterprise-grade security, compliance controls, and managed vulnerability remediation. + +Determinate Nix's [top-level flake](https://github.com/DeterminateSystems/nix-src/blob/main/flake.nix) still builds against upstream Nixpkgs. The flake `github:DeterminateSystems/nix-src?dir=packaging/secure-packages` uses Determinate Secure Packages. + +PR: [DeterminateSystems/nix-src#288](https://github.com/DeterminateSystems/nix-src/pull/288) + +## Flake schema changes + +Flakes can now distinguish between schemas for their own outputs (via the `schemas` output), and schemas that they make available to other flakes (via the `exportedSchemas` output). If you're using the `flake-schemas` flake, you should use its `exportedSchemas` output, e.g. +```nix +{ + inputs.flake-schemas.url = "https://flakehub.com/f/DeterminateSystems/flake-schemas/0.5"; + + outputs = { self, flake-schemas, ... }: { + schemas = flake-schemas.exportedSchemas // { ... my own schemas ... } + }; +} +``` + +PRs: [DeterminateSystems/nix-src#479](https://github.com/DeterminateSystems/nix-src/pull/479), [DeterminateSystems/nix-src#486](https://github.com/DeterminateSystems/nix-src/pull/486), +[DeterminateSystems/flake-schemas#63](https://github.com/DeterminateSystems/flake-schemas/pull/63) + +## Various bug fixes + +* Stop leaked logger in Finally block by @cole-h in [DeterminateSystems/nix-src#472](https://github.com/DeterminateSystems/nix-src/pull/472) +* Denote empty files in stack traces by @edolstra in [DeterminateSystems/nix-src#473](https://github.com/DeterminateSystems/nix-src/pull/473) +* Input::isRelative(): Fix assertion failure by @edolstra in [DeterminateSystems/nix-src#474](https://github.com/DeterminateSystems/nix-src/pull/474) +* Mark thread-local variables as initial-exec by @edolstra in [DeterminateSystems/nix-src#476](https://github.com/DeterminateSystems/nix-src/pull/476) +* Register temp roots in a few places by @edolstra in [DeterminateSystems/nix-src#482](https://github.com/DeterminateSystems/nix-src/pull/482) +* Sync with upstream 2.34.7 by @edolstra in [DeterminateSystems/nix-src#483](https://github.com/DeterminateSystems/nix-src/pull/483) +* Add a test for `nix ps` by @edolstra in [DeterminateSystems/nix-src#484](https://github.com/DeterminateSystems/nix-src/pull/484) +* nix develop: Restore hiding the "dirty" notice by @edolstra in [DeterminateSystems/nix-src#485](https://github.com/DeterminateSystems/nix-src/pull/485) + +**Full Changelog**: [v3.21.0...v3.21.1](https://github.com/DeterminateSystems/nix-src/compare/v3.21.0...v3.21.1) diff --git a/doc/manual/source/release-notes-determinate/v3.21.2.md b/doc/manual/source/release-notes-determinate/v3.21.2.md new file mode 100644 index 000000000000..caab2d1bd4d9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.2.md @@ -0,0 +1,52 @@ +# Release 3.21.2 (2026-06-18) + +* Based on [upstream Nix 2.34.7](../release-notes/rl-2.34.md). + +## Performance improvements + +`nix store optimise` is now multi-threaded. This makes it much faster (e.g. from 301s to 111s on one particular store) since hashing can be done in parallel. PR: [DeterminateSystems/nix-src#492](https://github.com/DeterminateSystems/nix-src/pull/492) + +Copying of sources to the Nix store is now cached better in the evaluator. PR: [DeterminateSystems/nix-src#507](https://github.com/DeterminateSystems/nix-src/pull/507) + +## Improved error messages + +When a user isn't allowed to connect to the Nix daemon, previously they would get the unhelpful error message: +``` +error: cannot open connection to remote store 'daemon': error: read of 32768 bytes: Connection reset by peer +``` + +Now they get: +``` +error: cannot open connection to remote store 'daemon': access denied by the Nix daemon (check the `allowed-users` setting in the daemon) +``` + +PR: [DeterminateSystems/nix-src#498](https://github.com/DeterminateSystems/nix-src/pull/498) + +## Various bug fixes + +* Add missing manpages by @edolstra in [DeterminateSystems/nix-src#490](https://github.com/DeterminateSystems/nix-src/pull/490) +* SQLiteStmt::Use: Rename operator() -> apply() by @edolstra in [DeterminateSystems/nix-src#491](https://github.com/DeterminateSystems/nix-src/pull/491) +* Fix running hydraJobs in CI by @edolstra in [DeterminateSystems/nix-src#494](https://github.com/DeterminateSystems/nix-src/pull/494) +* ci: include output paths in FlakeHub upload by @cole-h in [DeterminateSystems/nix-src#495](https://github.com/DeterminateSystems/nix-src/pull/495) +* fixup: don't include linux-only drvs if not on linux by @cole-h in [DeterminateSystems/nix-src#496](https://github.com/DeterminateSystems/nix-src/pull/496) +* Daemon: Ignore interrupts between accept() and child start by @edolstra in [DeterminateSystems/nix-src#502](https://github.com/DeterminateSystems/nix-src/pull/502) +* nario tweaks by @edolstra in [DeterminateSystems/nix-src#499](https://github.com/DeterminateSystems/nix-src/pull/499) +* Fix crash in builtins.break by @edolstra in [DeterminateSystems/nix-src#505](https://github.com/DeterminateSystems/nix-src/pull/505) +* Fix SSH connection sharing by @pjungkamp in [DeterminateSystems/nix-src#506](https://github.com/DeterminateSystems/nix-src/pull/506) +* Replace select() by poll() by @edolstra in [DeterminateSystems/nix-src#504](https://github.com/DeterminateSystems/nix-src/pull/504) +* LocalStore::doAddToStore(): Check NAR size before NAR hash by @proski in [DeterminateSystems/nix-src#512](https://github.com/DeterminateSystems/nix-src/pull/512) +* Increase the sourceToSink/sinkToSource coroutine stack sizes by @xokdvium in [DeterminateSystems/nix-src#511](https://github.com/DeterminateSystems/nix-src/pull/511) +* Fix LocalStore::addMultipleToStore SinkToSource destructor unsoundness by @lilyinstarlight in [DeterminateSystems/nix-src#513](https://github.com/DeterminateSystems/nix-src/pull/513) +* Fix exceptions not being caught in nix::Pid::~Pid by @lilyinstarlight in [DeterminateSystems/nix-src#516](https://github.com/DeterminateSystems/nix-src/pull/516) + +## Refactorings + +* Binary cache info by @edolstra in [DeterminateSystems/nix-src#493](https://github.com/DeterminateSystems/nix-src/pull/493) + +## Release process improvements + +* Build all the archs without DSP by @grahamc in [DeterminateSystems/nix-src#503](https://github.com/DeterminateSystems/nix-src/pull/503) +* Mirror to nix-artifacts on release of tags by @grahamc in [DeterminateSystems/nix-src#501](https://github.com/DeterminateSystems/nix-src/pull/501) +* ci: fixup mirroring by @cole-h in [DeterminateSystems/nix-src#515](https://github.com/DeterminateSystems/nix-src/pull/515) + +**Full Changelog**: [v3.21.1...v3.21.2](https://github.com/DeterminateSystems/nix-src/compare/v3.21.1...v3.21.2) diff --git a/doc/manual/source/release-notes-determinate/v3.21.5.md b/doc/manual/source/release-notes-determinate/v3.21.5.md new file mode 100644 index 000000000000..68a58d8f8776 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.5.md @@ -0,0 +1,39 @@ +# Release 3.21.5 (2026-07-08) + +* Based on [upstream Nix 2.34.8](../release-notes/rl-2.34.md). + +## Binary cache performance improvements + +This release includes a number of async rewrites from upstream Nix that make querying missing paths much faster. PR: [DeterminateSystems/nix-src#514](https://github.com/DeterminateSystems/nix-src/pull/514) + +## Remote store performance improvements + +Registering temporary roots on remote stores (e.g. when doing `nix copy` to a remote store over `ssh-ng`) is now much faster. PR: [DeterminateSystems/nix-src#538](https://github.com/DeterminateSystems/nix-src/pull/538) + +Computing closures (e.g. `nix-store -qR`) is now much faster on high-latency `ssh-ng` stores. PR: [DeterminateSystems/nix-src#539](https://github.com/DeterminateSystems/nix-src/pull/539) + +## Parallel evaluation performance improvements + +This release delivers a number of performance improvements to parallel evaluation. + +PRs: [DeterminateSystems/nix-src#531](https://github.com/DeterminateSystems/nix-src/pull/531), [DeterminateSystems/nix-src#535](https://github.com/DeterminateSystems/nix-src/pull/535), [DeterminateSystems/nix-src#536](https://github.com/DeterminateSystems/nix-src/pull/536), [DeterminateSystems/nix-src#537](https://github.com/DeterminateSystems/nix-src/pull/537), [DeterminateSystems/nix-src#546](https://github.com/DeterminateSystems/nix-src/pull/546) + +## Miscellaneous improvements + +* `nix search` now prints the total number of derivations, in addition to the number of matching derivations. PR: [DeterminateSystems/nix-src#532](https://github.com/DeterminateSystems/nix-src/pull/532) + +## Bug fixes and other changes + +* Allow local networking in libmicrohttpd and functional tests by @lilyinstarlight in [DeterminateSystems/nix-src#518](https://github.com/DeterminateSystems/nix-src/pull/518) +* Don't ignore substitution failures for BasicDerivation inputSrcs by @edolstra in [DeterminateSystems/nix-src#520](https://github.com/DeterminateSystems/nix-src/pull/520) +* Sync with upstream 2.34-maintenance by @edolstra in [DeterminateSystems/nix-src#521](https://github.com/DeterminateSystems/nix-src/pull/521) +* fixup: make binary cache nuclear option more general by @cole-h in [DeterminateSystems/nix-src#524](https://github.com/DeterminateSystems/nix-src/pull/524) +* NarInfoDiskCache: Support arbitrary nix-cache-info fields by @edolstra in [DeterminateSystems/nix-src#525](https://github.com/DeterminateSystems/nix-src/pull/525) +* ci: update DSP on releases by @cole-h in [DeterminateSystems/nix-src#526](https://github.com/DeterminateSystems/nix-src/pull/526) +* Add some calls to addTempRoot() by @edolstra in [DeterminateSystems/nix-src#527](https://github.com/DeterminateSystems/nix-src/pull/527) +* flake.lock: Update by @edolstra in [DeterminateSystems/nix-src#529](https://github.com/DeterminateSystems/nix-src/pull/529) + +## New Contributors +* @lilyinstarlight made their first contribution in [DeterminateSystems/nix-src#518](https://github.com/DeterminateSystems/nix-src/pull/518) + +**Full Changelog**: [v3.21.2...v3.21.5](https://github.com/DeterminateSystems/nix-src/compare/v3.21.2...v3.21.5) diff --git a/doc/manual/source/release-notes-determinate/v3.21.6.md b/doc/manual/source/release-notes-determinate/v3.21.6.md new file mode 100644 index 000000000000..0c3b32c23dc4 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.6.md @@ -0,0 +1,33 @@ +# Release 3.21.6 (2026-07-14) + +* Based on [upstream Nix 2.34.8](../release-notes/rl-2.34.md). + +## Parallel evaluation improvements + +`NIX_COUNT_CALLS` is now compatible with parallel evaluation. In addition, we fixed a bug that prevented it from reporting attribute access statistics. PR: [DeterminateSystems/nix-src#562](https://github.com/DeterminateSystems/nix-src/pull/562) + +We improved parallel evaluation performance by allocating more blocks per acquisition of the global Boehm GC lock. PR: [DeterminateSystems/nix-src#547](https://github.com/DeterminateSystems/nix-src/pull/547) + +## Bug fixes + +We made several bug fixes to address Sentry crash reports. PRs: [DeterminateSystems/nix-src#558](https://github.com/DeterminateSystems/nix-src/pull/558), [DeterminateSystems/nix-src#560](https://github.com/DeterminateSystems/nix-src/pull/560), [DeterminateSystems/nix-src#561](https://github.com/DeterminateSystems/nix-src/pull/561), [DeterminateSystems/nix-src#563](https://github.com/DeterminateSystems/nix-src/pull/563) + +Other bug fixes: + +* Fix data race initializing AttrCursor::_value by @edolstra in [DeterminateSystems/nix-src#557](https://github.com/DeterminateSystems/nix-src/pull/557) +* On Linux, don't hang if the sandbox helper dies before opening the pty slave by @edolstra in [DeterminateSystems/nix-src#559](https://github.com/DeterminateSystems/nix-src/pull/559) + +## Release process improvements + +* libutil: Fix GCC build with -Wchanges-meaning by @edolstra in [DeterminateSystems/nix-src#549](https://github.com/DeterminateSystems/nix-src/pull/549) +* fixup: build non-dsp nix on tags as well by @cole-h in [DeterminateSystems/nix-src#550](https://github.com/DeterminateSystems/nix-src/pull/550) +* build.yml: Enable provenance by @edolstra in [DeterminateSystems/nix-src#552](https://github.com/DeterminateSystems/nix-src/pull/552) +* Drop soversion by @edolstra in [DeterminateSystems/nix-src#555](https://github.com/DeterminateSystems/nix-src/pull/555) +* Restore uploading of debug symbols for build_x86_64-linux_no_dsp by @edolstra in [DeterminateSystems/nix-src#556](https://github.com/DeterminateSystems/nix-src/pull/556) + +## Miscellaneous changes + +* Root values cleanup by @edolstra in [DeterminateSystems/nix-src#554](https://github.com/DeterminateSystems/nix-src/pull/554) +* Add initial CLAUDE.md by @edolstra in [DeterminateSystems/nix-src#551](https://github.com/DeterminateSystems/nix-src/pull/551) + +**Full Changelog**: [v3.21.5...v3.21.6](https://github.com/DeterminateSystems/nix-src/compare/v3.21.5...v3.21.6) diff --git a/doc/manual/source/release-notes-determinate/v3.21.7.md b/doc/manual/source/release-notes-determinate/v3.21.7.md new file mode 100644 index 000000000000..136a3e57c08a --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.7.md @@ -0,0 +1,13 @@ +# Release 3.21.7 (2026-07-15) + +* Based on [upstream Nix 2.34.8](../release-notes/rl-2.34.md). + +## Performance improvement + +`nix search` is slightly faster when using parallel evaluation. PR: [DeterminateSystems/nix-src#564](https://github.com/DeterminateSystems/nix-src/pull/564) + +## Bug fix + +We improved on a bug fix in the previous release to prevent crashes in OpenSSL when Nix exits. PR: [DeterminateSystems/nix-src#566](https://github.com/DeterminateSystems/nix-src/pull/566) + +**Full Changelog**: [v3.21.6...v3.21.7](https://github.com/DeterminateSystems/nix-src/compare/v3.21.6...v3.21.7) diff --git a/doc/manual/source/release-notes-determinate/v3.21.8.md b/doc/manual/source/release-notes-determinate/v3.21.8.md new file mode 100644 index 000000000000..5285f5898f1d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.8.md @@ -0,0 +1,11 @@ +# Release 3.21.8 (2026-07-20) + +* Based on [upstream Nix 2.34.8](../release-notes/rl-2.34.md). + +## Bug fixes +* Fix libgit2 ll_find_delta leaking worker threads by @edolstra in [DeterminateSystems/nix-src#568](https://github.com/DeterminateSystems/nix-src/pull/568) +* Don't report SIGQUIT to Sentry by @edolstra in [DeterminateSystems/nix-src#570](https://github.com/DeterminateSystems/nix-src/pull/570) +* Don't abort if NIX_GET_COMPLETIONS is out of range by @edolstra in [DeterminateSystems/nix-src#571](https://github.com/DeterminateSystems/nix-src/pull/571) + + +**Full Changelog**: [v3.21.7...v3.21.8](https://github.com/DeterminateSystems/nix-src/compare/v3.21.7...v3.21.8) diff --git a/doc/manual/source/release-notes-determinate/v3.21.9.md b/doc/manual/source/release-notes-determinate/v3.21.9.md new file mode 100644 index 000000000000..91a479bf342e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.21.9.md @@ -0,0 +1,9 @@ +# Release 3.21.9 (2026-07-30) + +* Based on [upstream Nix 2.34.8](../release-notes/rl-2.34.md). + +## Bug fixes +* Populate the srcToStore cache when we have a fetcher cache hit by @edolstra in [DeterminateSystems/nix-src#575](https://github.com/DeterminateSystems/nix-src/pull/575) + + +**Full Changelog**: [v3.21.8...v3.21.9](https://github.com/DeterminateSystems/nix-src/compare/v3.21.8...v3.21.9) diff --git a/doc/manual/source/release-notes-determinate/v3.8.6.md b/doc/manual/source/release-notes-determinate/v3.8.6.md new file mode 100644 index 000000000000..8f917f2362ff --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.8.6.md @@ -0,0 +1,14 @@ +# Release 3.8.6 (2025-08-19) + +* Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +## What's Changed +* Auto update release notes by @grahamc in [DeterminateSystems/nix-src#170](https://github.com/DeterminateSystems/nix-src/pull/170) +* Use WAL mode for SQLite cache databases (2nd attempt) by @edolstra in [DeterminateSystems/nix-src#167](https://github.com/DeterminateSystems/nix-src/pull/167) +* Enable parallel marking in boehm-gc by @edolstra in [DeterminateSystems/nix-src#168](https://github.com/DeterminateSystems/nix-src/pull/168) +* BasicClientConnection::queryPathInfo(): Don't throw exception for invalid paths by @edolstra in [DeterminateSystems/nix-src#172](https://github.com/DeterminateSystems/nix-src/pull/172) +* Fix queryPathInfo() negative caching by @edolstra in [DeterminateSystems/nix-src#173](https://github.com/DeterminateSystems/nix-src/pull/173) +* forceDerivation(): Wait for async path write after forcing value by @edolstra in [DeterminateSystems/nix-src#176](https://github.com/DeterminateSystems/nix-src/pull/176) + + +**Full Changelog**: [v3.8.5...v3.8.6](https://github.com/DeterminateSystems/nix-src/compare/v3.8.5...v3.8.6) diff --git a/doc/manual/source/release-notes-determinate/v3.9.0.md b/doc/manual/source/release-notes-determinate/v3.9.0.md new file mode 100644 index 000000000000..66deb69b6192 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.9.0.md @@ -0,0 +1,45 @@ +# Release 3.9.0 (2025-08-26) + +* Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +## What's Changed + +### Build-time flake inputs + +Some of our users have hundreds or thousands of flake inputs. +In those cases, it is painfully slow for Nix to fetch all the inputs during evaluation of the flake. + +Determinate Nix has an experimental feature for deferring the fetching to build time of the dependent derivations. + +This is currently in developer preview. +If you would like to try it, add the experimental feature to your `/etc/nix/nix.custom.conf`: + +```ini +extra-experimental-features = build-time-fetch-tree +``` + +Then, mark an input to be fetched at build time: + +```nix +inputs.example = { + type = "github"; + owner = "DeterminateSystems"; + repo = "example"; + flake = false; # <-- currently required + buildTime = true; +}; +``` + +Let us know what you think! + +PR: [DeterminateSystems/nix-src#49](https://github.com/DeterminateSystems/nix-src/pull/49) + +### Corrected inconsistent behavior of `nix flake check` + +Users reported that `nix flake check` would not consistently validate the entire flake. + +We've fixed this issue and improved our testing around `nix flake check`. + +PR: [DeterminateSystems/nix-src#182](https://github.com/DeterminateSystems/nix-src/pull/182) + +**Full Changelog**: [v3.8.6...v3.9.0](https://github.com/DeterminateSystems/nix-src/compare/v3.8.6...v3.9.0) diff --git a/doc/manual/source/release-notes-determinate/v3.9.1.md b/doc/manual/source/release-notes-determinate/v3.9.1.md new file mode 100644 index 000000000000..38d17199c2c0 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.9.1.md @@ -0,0 +1,20 @@ +# Release 3.9.1 (2025-08-28) + +- Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +### A useful `nix flake init` template default + +Nix's default flake template is [extremely bare bones](https://github.com/NixOS/templates/blob/ad0e221dda33c4b564fad976281130ce34a20cb9/trivial/flake.nix), and not a useful starting point. + +Deteminate Nix now uses [a more fleshed out default template](https://github.com/DeterminateSystems/flake-templates/blob/8af99b99627da41f16897f60eb226db30c775e76/default/flake.nix), including targeting multiple systems. + +PR: [DeterminateSystems/nix-src#180](https://github.com/DeterminateSystems/nix-src/pull/180) + +### Build cancellation is repaired on macOS + +A recent macOS update changed how signals are handled by Nix and broke using Ctrl-C to stop a build. +Determinate Nix on macOS correctly handles these signals and stops the build. + +PR: [DeterminateSystems/nix-src#184](https://github.com/DeterminateSystems/nix-src/pull/184) + +**Full Changelog**: [v3.9.0...v3.9.1](https://github.com/DeterminateSystems/nix-src/compare/v3.9.0...v3.9.1) diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 813279c9f05f..0596ef909619 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -69,7 +69,7 @@ This makes it match `nix derivation show`, which also maps store paths to information. -- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) +- When Nix is installed using the binary installer, in supported shells (Bash, Zsh, Fish) [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3c3..92e5f4599440 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index e9b46bb22b70..f608fb54f7d9 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -268,6 +268,21 @@ be configured using the `warn-large-path-threshold` setting, e.g. `--warn-large-path-threshold 100M`. +- Wrap filesystem exceptions more correctly [#11378](https://github.com/NixOS/nix/pull/11378) + + With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions. + + This led to no longer generating error traces, for example when listing a non-existing directory. + + This version catches these types of exception correctly and wraps them into Nix's own exeception type. + + Author: [**@Mic92**](https://github.com/Mic92) + +- `` uses TLS verification [#11585](https://github.com/NixOS/nix/pull/11585) + + Previously `` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `` will now fail if you're fetching from a HTTPS server that does not have a valid certificate. + + `` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue. ## Contributors diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4c6..5a65ed99af29 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9ebd..c59ecd6c2456 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed697029389..0849f097cd0e 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. @@ -279,3 +279,35 @@ This release was made possible by the following 33 contributors: - Henry [**(@cootshk)**](https://github.com/cootshk) - Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg) - Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +# Release 2.33.3 (2026-02-13) + +- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) + + S3 binary caches now use virtual-hosted-style URLs + (`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs + (`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS + S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection + exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads. + + A new `addressing-style` store option controls this behavior: + + - `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style + for custom endpoints. + - `path`: forces path-style addressing (deprecated by AWS). + - `virtual`: forces virtual-hosted-style addressing (bucket names must not + contain dots). + + Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back + to path-style addressing in `auto` mode, because dotted names create + multi-level subdomains that break TLS wildcard certificate validation. + + Example using path-style for backwards compatibility: + + ``` + s3://my-bucket/key?region=us-east-1&addressing-style=path + ``` + + Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing + idle connections from being silently dropped by intermediate network devices + (NATs, firewalls, load balancers). + diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index 89017f1767cf..91ee4ad5e9af 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -119,7 +119,7 @@ New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket. It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`. - The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting. + The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-local-store-use-roots-daemon) store setting. This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store). diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 000000000000..a712b6385627 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d208d..cb9cb3099680 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e800129342..087413406487 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,72 +1,178 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs after the builder has exited. + + The builder process on exit should have left behind files for each output the derivation is supposed to produce. + The files must be processed to turn them into bona fide store objects. + If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). +In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs @@ -74,28 +180,40 @@ If the builder exited successfully, the following steps happen in order to turn - **Normalize the file permissions** - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. - - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + + Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. + + The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). + (This is not a special restriction for outputs; it is true for the references of all store objects in general.) + + In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. + For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. + Scratch store paths must therefore be used instead. + Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index aa65fbe4932a..9fadcff3c6ca 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -13,7 +13,7 @@ Given the method, the output's name (computed from the derivation name and outpu ## Fixed-output content-addressing {#fixed} In this case the content address of the *fixed* in advanced by the derivation itself. -In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. +In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must match that given in the derivation, or the build of that derivation will be deemed a failure. The output spec for an output with a fixed content addresses additionally contains: diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b04..9b46405cdf58 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d724..6df9b94961e8 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e572063..2ffad5f0ec94 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 000000000000..6ce21f7788c9 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f1402988..d063fc4fdc05 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 71ec772fb528..eb84ab84370d 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -42,9 +42,9 @@ The *referrers closure* of a store object are the store objects that can reach t > - The references of a store object --- the set of store paths called the references --- is a field of a store object, and thus intrinsic by definition. Regardless of what store contains the store object in question, and what else that store may or may not contain, the references are the same. > -> - The requisites of a store object are almost intrinsic --- some store paths due not precisely refer to a unique single store object. +> - The requisites of a store object are almost intrinsic --- some store paths do not precisely refer to a unique single store object. > Exactly what store object is being referenced, and what in turn *its* references are, depends on the store in question. -> Different stores that disagree. +> Different stores may disagree on what a given store path refers to. > > - The referrers of a store object are completely extrinsic, and depends solely on the store which contains that store object, not the store object itself. > Other store objects which refer to the store object in question may be added or removed from the store. @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 7834ac510271..282b7545a231 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. @@ -63,9 +63,9 @@ That provisional store path is instead what is replaced with the sentinel value, > If, conversely, the data after this normalization process is still different, we'll compute a different content-address. > The method of preparing the provisional self-referenced data has *failed* to be deterministic in the sense of not *leaking* the choice of provisional store path --- a choice which is supposed to be arbitrary --- into the final store object. > -> This property is informal because at this stage, we are just described store objects, which have no formal notion of their origin. -> Without such a formal notion, there is nothing to formally accuse of being insufficiently deterministic. -> Where we cover [derivations](@docroot@/store/derivation/index.md), we will have a chance to make this a formal property, not of content-addressed store objects themselves, but of derivations that *produce* content-addressed store objects. +> This property is informal at this stage, because we are just describing store objects, which have no formal notion of their origin. +> Without that, we cannot rigorously define what it would mean for the process to be insufficiently deterministic. +> When we cover [derivations](@docroot@/store/derivation/index.md), we will be able to make this a formal property — not of content-addressed store objects themselves, but of the derivations that *produce* them. ### Name and Store Directory diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004c2..43037f7baac6 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/doc/manual/source/store/types/index.md.in b/doc/manual/source/store/types/index.md.in index b211ac98fe3a..fbdcf0111f59 100644 --- a/doc/manual/source/store/types/index.md.in +++ b/doc/manual/source/store/types/index.md.in @@ -26,6 +26,8 @@ tells Nix to access the store on a remote machine via the SSH protocol, using `/path/to/my/key` as the SSH private key. The supported settings for each store type are documented below. +### `auto` special URL {#auto} + The special store URL `auto` causes Nix to automatically select a store as follows: @@ -35,7 +37,7 @@ store as follows: * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](./local-store.md) +* Otherwise, on Linux only, use the [local chroot store](@docroot@/store/types/local-store.md#chroot) `~/.local/share/nix/root`, which will be created automatically if it does not exist. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777f7..40bfef7d2f8f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/docker.nix b/docker.nix index d85927a08020..2bd6751762fa 100644 --- a/docker.nix +++ b/docker.nix @@ -8,6 +8,7 @@ # Image configuration name ? "nix", tag ? "latest", + fromImage ? null, bundleNixpkgs ? true, channelName ? "nixpkgs", channelURL ? "https://channels.nixos.org/nixpkgs-unstable", @@ -27,6 +28,8 @@ "org.opencontainers.image.description" = "Nix container image"; }, Cmd ? [ (lib.getExe bashInteractive) ], + extraPrePaths ? [ ], + extraPostPaths ? [ ], # Default Packages nix ? pkgs.nix, bashInteractive ? pkgs.bashInteractive, @@ -336,7 +339,7 @@ let globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" ln -s ${flake-registry-path} $out$globalFlakeRegistryPath mkdir -p $out/nix/var/nix/gcroots/auto - rootName=$(${lib.getExe' nix "nix"} --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + rootName=$(${lib.getExe' nix "nix"} hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName '') ); @@ -352,6 +355,7 @@ dockerTools.buildLayeredImageWithNixDb { gid uname gname + fromImage ; contents = [ baseSystem ]; @@ -372,11 +376,15 @@ dockerTools.buildLayeredImageWithNixDb { Env = [ "USER=${uname}" "PATH=${ - lib.concatStringsSep ":" [ - "${userHome}/.nix-profile/bin" - "/nix/var/nix/profiles/default/bin" - "/nix/var/nix/profiles/default/sbin" - ] + lib.concatStringsSep ":" ( + extraPrePaths + ++ [ + "${userHome}/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ] + ++ extraPostPaths + ) }" "MANPATH=${ lib.concatStringsSep ":" [ diff --git a/flake.lock b/flake.lock index 4c0bf91927a4..99e0a8ec1cce 100644 --- a/flake.lock +++ b/flake.lock @@ -23,55 +23,50 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", - "type": "github" + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "revCount": 480, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.480%2Brev-17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e/019f2195-dee5-7233-9747-eca0c27f7406/source.tar.gz" }, "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" } }, "git-hooks-nix": { "inputs": { - "flake-compat": [], - "gitignore": [], + "flake-compat": "flake-compat", "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", - "type": "github" + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "revCount": 1231, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1231%2Brev-43b3c1ab9d40fb1dbb008f451988a91e375825e9/019f7135-8fdf-76f0-b1a1-d2c67e91af8d/source.tar.gz" }, "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941" } }, "nixpkgs": { "locked": { - "lastModified": 1771903837, - "narHash": "sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E=", - "rev": "e764fc9a405871f1f6ca3d1394fb422e0a0c3951", + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "revCount": 1009383, "type": "tarball", - "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1009383%2Brev-4382ed2b7a6839d4280a9b386db49cbc5907414d/019f6c11-ad78-7500-a194-d18a5bad0fbe/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz" + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2605" } }, "nixpkgs-23-11": { @@ -108,7 +103,6 @@ }, "root": { "inputs": { - "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", diff --git a/flake.nix b/flake.nix index 6bb0c609f6d7..d46b4683146a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,25 +1,17 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2605"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; - inputs.flake-compat = { - url = "github:NixOS/flake-compat"; - flake = false; - }; # dev tooling - inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + inputs.flake-parts.url = "https://flakehub.com/f/hercules-ci/flake-parts/0.1"; + inputs.git-hooks-nix.url = "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; - # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.git-hooks-nix.inputs.flake-compat.follows = ""; - inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ @@ -34,26 +26,24 @@ officialRelease = true; - linux32BitSystems = [ "i686-linux" ]; + linux32BitSystems = [ ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux32BitSystems ++ linux64BitSystems; darwinSystems = [ - "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-unknown-linux-gnueabihf" - "armv7l-unknown-linux-gnueabihf" - "riscv64-unknown-linux-gnu" + #"armv6l-unknown-linux-gnueabihf" + #"armv7l-unknown-linux-gnueabihf" + #"riscv64-unknown-linux-gnu" # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 # "x86_64-unknown-netbsd" - "x86_64-unknown-freebsd" - "x86_64-w64-mingw32" + #"x86_64-unknown-freebsd" ]; stdenvs = [ @@ -113,11 +103,8 @@ { config = crossSystem; } - // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { - useLLVM = true; - } // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { - emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; + emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable}/bin/wine"; }; overlays = [ (overlayFor (pkgs: pkgs.${stdenv})) @@ -331,12 +318,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -375,6 +356,40 @@ nix-manual-manpages-only = nixpkgsFor.${system}.native.nixComponents2.nix-manual-manpages-only; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs; + + fallbackPathsNix = + let + pkgs = nixpkgsFor.${system}.native; + + closures = forAllSystems (system: self.packages.${system}.default.outPath); + + closures_json = + pkgs.runCommand "versions.json" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "json" ]; + json = builtins.toJSON closures; + } + '' + cat "$jsonPath" | jq . > $out + ''; + + closures_nix = + pkgs.runCommand "versions.nix" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "template" ]; + jsonPath = closures_json; + template = '' + builtins.fromJSON('''@closures@''') + ''; + } + '' + export closures=$(cat "$jsonPath"); + substituteAll "$templatePath" "$out" + ''; + in + closures_nix; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // @@ -425,7 +440,7 @@ supportsCross = false; }; - "nix-perl-bindings" = { + "nix-clang-tidy-plugin" = { supportsCross = false; }; } @@ -439,8 +454,6 @@ { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; - "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents2.${pkgName}; } // flatMapAttrs (lib.genAttrs stdenvs (_: { })) ( stdenvName: @@ -464,7 +477,19 @@ } ) ) + // lib.optionalAttrs (linuxOnly -> nixpkgsFor.${system}.native.stdenv.hostPlatform.isLinux) { + "${pkgName}-static" = + let + pkgs = nixpkgsFor.${system}; + in + ( + if pkgs.native.stdenv.hostPlatform.isDarwin then pkgs.nativeForStdenv.libcxxStdenv else pkgs.native + ).pkgsStatic.nixComponents2.${pkgName}; + } ) + // lib.optionalAttrs (self.hydraJobs.rustInstaller ? ${system}) { + rustInstaller = self.hydraJobs.rustInstaller.${system}; + } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let @@ -508,6 +533,16 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + makeShell' = + { pkgs }: + makeShell { + inherit pkgs; + nixComponents = pkgs.nixComponents2.overrideScope ( + finalScope: prevScope: { + withUnityBuild = false; + } + ); + }; prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; }); in forAllSystems ( @@ -515,37 +550,11 @@ prefixAttrs "native" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}; } ) ) - // lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) ( - prefixAttrs "static" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic; - } - ) - ) - // prefixAttrs "llvm" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsLLVM; - } - ) - ) - // prefixAttrs "cross" ( - forAllCrossSystems ( - crossSystem: - makeShell { - pkgs = nixpkgsFor.${system}.cross.${crossSystem}; - } - ) - ) - ) // { native = self.devShells.${system}.native-stdenv; default = self.devShells.${system}.native; diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index f742a7440023..84207a398a5d 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -16,6 +16,16 @@ # https://flake.parts/options/git-hooks-nix#options pre-commit.settings = { + # `self.outPath` from git-hooks-nix's flake-module is not path-normalized, + # so when this flake is evaluated via `packaging/secure-packages` (whose + # `inputs.nix.url = "../.."`), it expands to `.../source/packaging/secure-packages/../..`, + # which breaks the stdenv unpackPhase. Use a clean relative path instead. + rootSrc = lib.mkForce ( + builtins.path { + name = "source"; + path = ../.; + } + ); hooks = { # Conflicts are usually found by other checks, but not those in docs, # and potentially other places. @@ -37,30 +47,17 @@ fi ''}"; }; - meson-format = - let - meson = pkgs.meson.overrideAttrs { - doCheck = false; - doInstallCheck = false; - patches = [ - (pkgs.fetchpatch { - url = "https://github.com/mesonbuild/meson/commit/38d29b4dd19698d5cad7b599add2a69b243fd88a.patch"; - hash = "sha256-PgPBvGtCISKn1qQQhzBW5XfknUe91i5XGGBcaUK4yeE="; - }) - ]; - }; - in - { - enable = true; - files = "(meson.build|meson.options)$"; - entry = "${pkgs.writeScript "format-meson" '' - #!${pkgs.runtimeShell} - for file in "$@"; do - ${lib.getExe meson} format -ic ${../meson.format} "$file" - done - ''}"; - }; - nixfmt-rfc-style = { + meson-format = { + enable = true; + files = "(meson.build|meson.options)$"; + entry = "${pkgs.writeScript "format-meson" '' + #!${pkgs.runtimeShell} + for file in "$@"; do + ${lib.getExe pkgs.meson} format -ic ${../meson.format} "$file" + done + ''}"; + }; + nixfmt = { enable = true; excludes = [ # Invalid @@ -114,6 +111,7 @@ # Don't format vendored code ''^doc/manual/redirects\.js$'' ''^doc/manual/theme/highlight\.js$'' + ''^src/libfetchers/builtin-flake-registry\.json$'' ]; }; shellcheck = { diff --git a/maintainers/link-headers b/maintainers/link-headers new file mode 100755 index 000000000000..2457a2dc8295 --- /dev/null +++ b/maintainers/link-headers @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# This script must be run from the root of the Nix repository. +# +# For include path hygiene, we need to put headers in a separate +# directory than sources. But during development, it is nice to paths +# that are similar for headers and source files, e.g. +# `foo/bar/baz.{cc,hh}`, e.g. for less typing when opening one file, and +# then opening the other file. +# +# This script symlinks the headers next to the source files to +# facilitate such a development workflows. It also updates +# `.git/info/exclude` so that the symlinks are not accidentally committed +# by mistake. + +from pathlib import Path +import subprocess +import os + + +def main() -> None: + # Path to the source directory + GIT_TOPLEVEL = Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + ) + + # Get header files from git + result = subprocess.run( + ["git", "-C", str(GIT_TOPLEVEL), "ls-files", "*/include/nix/**.hh"], + text=True, + stdout=subprocess.PIPE, + check=True, + ) + header_files = result.stdout.strip().split("\n") + header_files.sort() + + links = [] + for file_str in header_files: + project_str, header_str = file_str.split("/include/nix/", 1) + project = Path(project_str) + header = Path(header_str) + + # Reconstruct the full path (relative to SRC_DIR) to the header file. + file = project / "include" / "nix" / header + + # The symlink should be created at "project/header", i.e. next to the project's sources. + link = project / header + + # Compute a relative path from the symlink's parent directory to the actual header file. + relative_source = os.path.relpath( + GIT_TOPLEVEL / file, GIT_TOPLEVEL / link.parent + ) + + # Create the symbolic link. + full_link_path = GIT_TOPLEVEL / link + full_link_path.parent.mkdir(parents=True, exist_ok=True) + if full_link_path.is_symlink(): + full_link_path.unlink() + full_link_path.symlink_to(relative_source) + links.append(link) + + # Generate .gitignore file + gitignore_path = GIT_TOPLEVEL / ".git" / "info" / "exclude" + gitignore_path.parent.mkdir(parents=True, exist_ok=True) + with gitignore_path.open("w") as gitignore: + gitignore.write("# DO NOT EDIT! Autogenerated\n") + gitignore.write( + "# Symlinks for headers to be next to sources for development\n" + ) + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + + for link in links: + gitignore.write(f"/{link}\n") + + +if __name__ == "__main__": + main() diff --git a/maintainers/release-process.md b/maintainers/release-process.md index f8b6b6bec572..aff0088a0d93 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -81,27 +81,33 @@ release: $ git push --set-upstream origin $VERSION-maintenance ``` -* Create a jobset for the release branch on Hydra as follows: - - * Go to the jobset of the previous release - (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). - - * Select `Actions -> Clone this jobset`. - - * Set identifier to `maintenance-$VERSION`. - - * Set description to `$VERSION release branch`. - - * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. - - * Hit `Create jobset`. - -* Wait for the new jobset to evaluate and build. If impatient, go to - the evaluation and select `Actions -> Bump builds to front of - queue`. - -* When the jobset evaluation has succeeded building, take note of the - evaluation ID (e.g. `1780832` in +* Create two jobsets for the release branch on Hydra: + + `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. + `maintenance-$VERSION-release` builds only the artifacts consumed by + `upload-release`, so a release can be cut without waiting on the full + matrix. The `-release` suffix keeps the pair adjacent in Hydra's + alphabetical jobset list and lets scripts derive one name from the + other. + + * Clone the previous `maintenance-*` jobset, set identifier + `maintenance-$VERSION`, description `$VERSION release branch`, flake + URL `github:NixOS/nix/$VERSION-maintenance`. + + * Clone the previous `maintenance-*-release` jobset (or create a new + **legacy** jobset), set identifier `maintenance-$VERSION-release`, + description `$VERSION release artifacts`, Nix expression + `packaging/release-jobs.nix` in input `src`, and add input `src` of + type *Git checkout* pointing at + `https://github.com/NixOS/nix $VERSION-maintenance`. + +* Wait for the `maintenance-$VERSION-release` jobset to evaluate and + build. If impatient, go to the evaluation and select `Actions -> Bump + builds to front of queue`. The aggregate job `release` turns green + once every required artifact is available. + +* When the release jobset evaluation has succeeded building, take note of + the evaluation ID (e.g. `1780832` in `https://hydra.nixos.org/eval/1780832`). * Tag the release: @@ -174,8 +180,9 @@ release: $ git push ``` -* Wait for the desired evaluation of the maintenance jobset to finish - building. +* Wait for the desired evaluation of the `maintenance-XX.YY-release` + jobset to finish building (the `release` aggregate job is the gating + signal). * Tag the release diff --git a/maintainers/upload-debug-info-to-sentry.py b/maintainers/upload-debug-info-to-sentry.py new file mode 100755 index 000000000000..87804d32a519 --- /dev/null +++ b/maintainers/upload-debug-info-to-sentry.py @@ -0,0 +1,164 @@ +#!/usr/bin/env nix +#!nix shell --inputs-from . nixpkgs#sentry-cli nixpkgs#python3 nixpkgs#binutils --command python3 + +import argparse +import json +import os +import platform +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +NAR_DIR = "/tmp/nars" +DEBUG_INFO_DIR = "/tmp/debug-info" + + +def get_dynamic_libraries(executable: str) -> list[str]: + if platform.system() == "Darwin": + result = subprocess.run(["otool", "-L", executable], capture_output=True, text=True, check=True) + libs = [] + for line in result.stdout.splitlines()[1:]: # skip first line (the binary path itself) + # otool -L output lines look like: + # /nix/store/.../libfoo.dylib (compatibility version X.Y.Z, current version A.B.C) + m = re.match(r"\s+(\S+)\s+\(", line) + if m: + libs.append(m.group(1)) + return libs + else: + result = subprocess.run(["ldd", executable], capture_output=True, text=True, check=True) + libs = [] + for line in result.stdout.splitlines(): + # ldd output lines look like: + # libfoo.so.1 => /nix/store/.../libfoo.so.1 (0x...) + # /lib64/ld-linux-x86-64.so.2 (0x...) + m = re.search(r"=> (/\S+)", line) + if m: + libs.append(m.group(1)) + elif line.strip().startswith("/"): + path = line.strip().split()[0] + libs.append(path) + return libs + + +def get_build_id(path: str) -> str | None: + result = subprocess.run(["readelf", "-n", path], capture_output=True, text=True) + m = re.search(r"Build ID:\s+([0-9a-f]+)", result.stdout) + return m.group(1) if m else None + + +def download_nar(build_id: str, archive: str) -> str: + """Download a NAR to /tmp/nars and return the local path. Skips if already present.""" + base_url = f"https://cache.nixos.org/debuginfo/{build_id}" + nar_url = urllib.parse.urljoin(base_url, archive) + filename = nar_url.split("/")[-1] + local_path = os.path.join(NAR_DIR, filename) + if not os.path.exists(local_path): + os.makedirs(NAR_DIR, exist_ok=True) + print(f" downloading {nar_url} ...", file=sys.stderr) + urllib.request.urlretrieve(nar_url, local_path) + else: + print(f" already have {filename}", file=sys.stderr) + return local_path + + +def extract_debug_symbols(nar_path: str, member: str, build_id: str) -> str: + """Extract a member from a .nar.xz into /tmp/debug-info/.debug. Returns the output path.""" + out_path = os.path.join(DEBUG_INFO_DIR, f"{build_id}.debug") + if os.path.exists(out_path): + print(f" already extracted {out_path}", file=sys.stderr) + return out_path + os.makedirs(DEBUG_INFO_DIR, exist_ok=True) + print(f" extracting {member} -> {out_path} ...", file=sys.stderr) + xz = subprocess.Popen(["xz", "-d"], stdin=open(nar_path, "rb"), stdout=subprocess.PIPE) + nar_cat = subprocess.run( + ["nix", "nar", "cat", "/dev/stdin", member], + stdin=xz.stdout, + capture_output=True, + check=True, + ) + xz.wait() + with open(out_path, "wb") as f: + f.write(nar_cat.stdout) + return out_path + + +def find_debug_file_in_dirs(build_id: str, debug_dirs: list[str]) -> str | None: + """Look for a .debug file by build ID under

/lib/debug/.build-id/NN/NNN.debug.""" + subpath = os.path.join("lib", "debug", ".build-id", build_id[:2], build_id[2:] + ".debug") + for d in debug_dirs: + candidate = os.path.join(d, subpath) + if os.path.exists(candidate): + return candidate + return None + + +def fetch_debuginfo(build_id: str) -> dict | None: + url = f"https://cache.nixos.org/debuginfo/{build_id}" + try: + with urllib.request.urlopen(url) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Upload debug symbols to Sentry." + ) + parser.add_argument("executable", help="Path to the executable (e.g. ./result/bin/nix)") + parser.add_argument("--project", help="Sentry project ID") + parser.add_argument("--debug-dir", action="append", default=[], metavar="DIR", + help="Directory to search for debug files (may be repeated, Linux only)") + args = parser.parse_args() + + libs = [args.executable] + get_dynamic_libraries(args.executable) + + if platform.system() == "Darwin": + # On macOS there are no separate debug info files; upload the binaries directly. + print("Files to upload:", file=sys.stderr) + for lib in libs: + print(f" {lib}", file=sys.stderr) + files_to_upload = libs + else: + debug_files = [] + print("ELF files to process:", file=sys.stderr) + for lib in libs: + debug_files.append(lib) + + build_id = get_build_id(lib) + if build_id is None: + print(f" {lib} (no build ID, uploading binary)", file=sys.stderr) + continue + + local = find_debug_file_in_dirs(build_id, args.debug_dir) + if local: + print(f" {lib} ({build_id}): found locally at {local}", file=sys.stderr) + debug_files.append(local) + continue + + debuginfo = fetch_debuginfo(build_id) + if debuginfo is None: + print(f" {lib} ({build_id}): no separate debug info", file=sys.stderr) + continue + + print(f" {lib} ({build_id}): member={debuginfo['member']}", file=sys.stderr) + nar_path = download_nar(build_id, debuginfo["archive"]) + debug_file = extract_debug_symbols(nar_path, debuginfo["member"], build_id) + debug_files.append(debug_file) + files_to_upload = debug_files + + if files_to_upload: + print(f"Uploading {len(files_to_upload)} file(s) to Sentry...", file=sys.stderr) + cmd = ["sentry-cli", "debug-files", "upload"] + if args.project: + cmd += ["--project", args.project] + subprocess.run(cmd + files_to_upload, check=True) + + +if __name__ == "__main__": + main() diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index f7678b7d1c0a..b618bd900d4e 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -64,7 +64,13 @@ sub fetch { #print Dumper($evalInfo); my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; -my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; +# Flake jobsets (`maintenance-X.Y`) expose the rev via the flake URL. +# The release-artifacts jobset (`maintenance-X.Y-release`) is a legacy +# jobset whose checkout is passed in as input `src`. +my $nixRev = ($flakeInfo + ? $flakeInfo->{revision} + : $evalInfo->{jobsetevalinputs}->{src}->{revision} + // $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix-everything.x86_64-linux", 'application/json')); #print Dumper($buildInfo); @@ -279,6 +285,10 @@ sub downloadFile { downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); }; warn "$@" if $@; + eval { + downloadFile("binaryTarballCross.x86_64-linux.x86_64-unknown-freebsd", "1"); + }; + warn "$@" if $@; downloadFile("installerScript", "1"); # Upload nix-fallback-paths.nix. @@ -290,6 +300,7 @@ sub downloadFile { " riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" . + " x86_64-freebsd = \"" . getStorePath("buildCross.nix-everything.x86_64-unknown-freebsd.x86_64-linux") . "\";\n" . "}\n"); for my $fn (glob "$tmpDir/*") { diff --git a/meson.build b/meson.build index 3e2d29cd5495..c01297fb5477 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -21,6 +18,14 @@ subproject('libflake') subproject('libmain') subproject('libcmd') +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libfetchers-c') +subproject('libexpr-c') +subproject('libflake-c') +subproject('libmain-c') + # Executables subproject('nix') @@ -37,21 +42,6 @@ if get_option('doc-gen') endif endif -# External C wrapper libraries -subproject('libutil-c') -subproject('libstore-c') -subproject('libfetchers-c') -subproject('libexpr-c') -subproject('libflake-c') -subproject('libmain-c') - -asan_enabled = 'address' in get_option('b_sanitize') - -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -63,7 +53,17 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif + +# Static Analysis +# Build the custom clang-tidy plugin if LLVM is available. +# required: false → gracefully skipped if dependency('LLVM') fails. +clang_tidy_plugin_sp = subproject('clang-tidy-plugin', required : false) +subdir('nix-meson-build-support/common/clang-tidy') diff --git a/meson.options b/meson.options index a306a84252ea..2e0d873fae08 100644 --- a/meson.options +++ b/meson.options @@ -15,10 +15,10 @@ option( ) option( - 'bindings', + 'functional-tests', type : 'boolean', value : true, - description : 'Build language bindings (e.g. Perl)', + description : 'Build functional (E2E) tests', ) option( diff --git a/nix-meson-build-support/common/asan-options/asan-options.cc b/nix-meson-build-support/common/asan-options/asan-options.cc index c9782fea03b5..62db7ddd20e0 100644 --- a/nix-meson-build-support/common/asan-options/asan-options.cc +++ b/nix-meson-build-support/common/asan-options/asan-options.cc @@ -1,4 +1,17 @@ -extern "C" [[gnu::retain, gnu::weak]] const char * __asan_default_options() +#if defined(__has_attribute) +# if __has_attribute(no_profile_instrument_function) +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION __attribute__((no_profile_instrument_function)) +# endif +#endif + +#ifndef NIX_NO_PROFILE_INSTRUMENT_FUNCTION +# define NIX_NO_PROFILE_INSTRUMENT_FUNCTION +#endif + +// This ASan hook is linked into many instrumented binaries and libraries. Do +// not emit coverage counters for it because repeated weak definitions of the +// hook can produce profile data that llvm-profdata treats as corrupted. +extern "C" [[gnu::retain, gnu::weak]] NIX_NO_PROFILE_INSTRUMENT_FUNCTION const char * __asan_default_options() { // We leak a bunch of memory knowingly on purpose. It's not worthwhile to // diagnose that memory being leaked for now. diff --git a/nix-meson-build-support/common/asan-options/meson.build b/nix-meson-build-support/common/asan-options/meson.build index 56e6a6a56a7f..80527b5a9884 100644 --- a/nix-meson-build-support/common/asan-options/meson.build +++ b/nix-meson-build-support/common/asan-options/meson.build @@ -1,7 +1,7 @@ # Clang gets grumpy about missing libasan symbols if -shared-libasan is not # passed when building shared libs, at least on Linux if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( - 'b_sanitize', + 'b_sanitize', )) add_project_link_arguments('-shared-libasan', language : 'cpp') endif diff --git a/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc b/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc index d9e34168bc97..292c57e7e4c7 100644 --- a/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc +++ b/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc @@ -1,4 +1,5 @@ #include "nix/util/error.hh" +#include "nix/util/sentry.hh" #include #include @@ -13,5 +14,6 @@ __wrap___assert_fail(const char * assertion, const char * file, unsigned int lin snprintf(buf, sizeof(buf), "Assertion '%s' failed in %s at %s:%" PRIuLEAST32, assertion, function, file, line); if (n < 0) nix::panic("Assertion failed and could not format error message"); - nix::panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); + nix::setSentryTag("assertion", buf); + nix::panic(buf); } diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy new file mode 100644 index 000000000000..b0cc92429c4e --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -0,0 +1,100 @@ +UseColor: true +Checks: + - -* + - bugprone-* + # Too many warnings + - -bugprone-assignment-in-if-condition + # Too many warnings + - -bugprone-narrowing-conversions + # Kind of nonsense + - -bugprone-easily-swappable-parameters + # Too many warnings for now + - -bugprone-implicit-widening-of-multiplication-result + # Exception handling patterns in Nix + - -bugprone-empty-catch + # Many warnings + - -bugprone-unchecked-optional-access + # Many warnings, questionable lint + - -bugprone-branch-clone + # Extremely noisy before clang 19: https://github.com/llvm/llvm-project/issues/93959 + - -bugprone-multi-level-implicit-pointer-conversion + # We don't compile out our asserts + - -bugprone-assert-side-effect + # TODO: figure out if this warning is useful + - -bugprone-exception-escape + # We use pointers to aggregates intentionally; void * would look weird + - -bugprone-sizeof-expression + # + # Checks disabled to pass on current codebase (can be progressively enabled): + # + # 11 warnings - optional value conversions in various places + - -bugprone-optional-value-conversion + # 4 warnings - switches without default cases + - -bugprone-switch-missing-default-case + # 4 warnings - string_view::data() usage patterns + - -bugprone-suspicious-stringview-data-usage + # 4 warnings - .cc files included in other files (intentional pattern) + - -bugprone-suspicious-include + # 2 warnings - unused return values (AllowCastToVoid helps but some remain) + - -bugprone-unused-return-value + # 2 warnings - unused local RAII-style variables + - -bugprone-unused-local-non-trivial-variable + # 2 warnings - returning const& from parameter + - -bugprone-return-const-ref-from-parameter + # 1 warning - signed char misuse + - -bugprone-signed-char-misuse + # 1 warning - calling parent virtual instead of override + - -bugprone-parent-virtual-call + # 1 warning - null termination issues + - -bugprone-not-null-terminated-result + # 1 warning - macro parentheses + - -bugprone-macro-parentheses + # 1 warning - increment/decrement in conditions + - -bugprone-inc-dec-in-conditions + # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) + - -bugprone-nondeterministic-pointer-iteration-order + # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) + - -bugprone-bitwise-pointer-cast + # 1 warning (header) - value.hh mkFailed: GC alloc in noexcept. Boehm's + # gc_cleanup::operator new isn't marked noexcept but aborts on OOM rather + # than throws; std::terminate here is the intended behavior anyway. + - -bugprone-unhandled-exception-at-new + # 1 warning (header) - fmt.hh Magenta::operator<<: generic colorizer + # template; fires when T=unsigned char but that instantiation is correct. + - -bugprone-unintended-char-ostream-output + # + # Non-bugprone checks (some disabled to pass on current codebase): + # + # 4 warnings - exceptions not derived from std::exception + # All thrown exceptions must derive from std::exception + # - hicpp-exception-baseclass + # 88 warnings - C-style casts should be explicit about intent + # - cppcoreguidelines-pro-type-cstyle-cast + # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) + # - cppcoreguidelines-avoid-capturing-lambda-coroutines + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 + # Custom nix checks (when added) + - nix-* + +CheckOptions: + # __asan_default_options: ASAN runtime configuration function (see nix-meson-build-support/common/asan-options/) + # __wrap___assert_fail: Linker-wrapped assert handler for better stack traces (see nix-meson-build-support/common/assert-fail/) + # _SingleDerivedPathRaw, _DerivedPathRaw: Internal type aliases in derived-path.hh (leading underscore pattern) + # _SingleBuiltPathRaw, _BuiltPathRaw: Internal type aliases in built-path.hh (leading underscore pattern) + bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' + # Allow explicitly discarding return values with (void) cast + bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py new file mode 100755 index 000000000000..24e4f290c607 --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Builds all generated files before running clang-tidy. + +clang-tidy needs all source files and headers to exist before it can +analyze the code. This script queries Ninja for all custom command +targets that generate files needed for analysis and builds them. + +Generated files include: +- .gen.hh: Embedded file headers (SQL schemas, Nix expressions, etc.) +- .gen.inc: Generated include files +- lexer-tab.cc, parser-tab.cc: Flex/Bison generated parsers +- Perl XS bindings, and other generated sources + +See: https://github.com/mesonbuild/meson/issues/12817 +""" + +import subprocess +from pathlib import Path + + +def get_targets_of_rule(build_root: str, rule_name: str) -> list[str]: + return ( + subprocess.check_output( + ["ninja", "-C", build_root, "-t", "targets", "rule", rule_name] + ) + .decode() + .strip() + .splitlines() + ) + + +def ninja_build(build_root: str, targets: list[str]): + if targets: + subprocess.check_call(["ninja", "-C", build_root, "--", *targets]) + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description="Build required targets for clang-tidy") + ap.add_argument("build_root", help="Ninja build root", type=str) + ap.add_argument("--stamp", type=Path, default=None, help="Stamp file to touch on success") + + args = ap.parse_args() + + custom_commands = get_targets_of_rule(args.build_root, "CUSTOM_COMMAND") + + targets = ( + # Generated headers from embedded files + [t for t in custom_commands if t.endswith(".gen.hh")] + # Generated include files + + [t for t in custom_commands if t.endswith(".gen.inc")] + # Flex/Bison generated parsers + + [t for t in custom_commands if t.endswith("-tab.cc")] + ) + ninja_build(args.build_root, targets) + + if args.stamp: + args.stamp.touch() + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py b/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py new file mode 100755 index 000000000000..8d7b99c7d438 --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/clang-tidy-runner.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Wrapper around run-clang-tidy with better UX. + +This script handles: +- Loading the custom nix-clang-tidy plugin +- Setting up header filters to only check project headers +- Managing parallelism based on available CPUs +- Working around run-clang-tidy's unpatched shebang (NixOS/nixpkgs#214945) +""" + +import multiprocessing +import os +import sys +from pathlib import Path + + +def default_concurrency(): + return min( + multiprocessing.cpu_count(), int(os.environ.get("NIX_BUILD_CORES", "16")) + ) + + +def go( + exe: str, + plugin_path: Path | None, + config_file: Path | None, + compile_commands_json_dir: Path, + jobs: int, + paths: list[Path], + werror: bool, + fix: bool, +): + args = [ + # clang-unwrapped ships run-clang-tidy with an unpatched '#!/usr/bin/env python3' + # shebang (see NixOS/nixpkgs#214945), which fails in sandboxed builds where + # /usr/bin/env doesn't exist. Invoke through sys.executable to bypass the shebang. + sys.executable, + exe, + "-quiet", + ] + + if plugin_path is not None: + args += ["-load", str(plugin_path)] + + if config_file is not None: + args += ["-config-file", str(config_file)] + + args += [ + "-p", + str(compile_commands_json_dir), + "-j", + str(jobs), + # Lint all project headers (public include/nix/*/ and private *.hh). + # Project headers are reached via relative -I flags; system/installed + # headers are absolute /nix/store/... paths. Match any non-absolute path + # so the filter works in both monolithic (../src/libfoo/include/...) + # and per-component (../include/...) build topologies. + "-header-filter", + r"^[^/].*\.hh", + # Exclude generated Bison/Flex headers (parser-tab.hh, lexer-tab.hh) + "-exclude-header-filter", + r".-tab\.hh$", + ] + + if werror: + args += ["-warnings-as-errors", "*"] + if fix: + args += ["-fix"] + + args += ["--"] + args += [str(p) for p in paths] + + os.execvp(sys.executable, args) + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description="Run clang-tidy on the Nix codebase") + ap.add_argument( + "--jobs", + "-j", + type=int, + default=default_concurrency(), + help="Parallel linting jobs to run", + ) + ap.add_argument( + "--plugin-path", + type=Path, + default=None, + help="Path to the nix-clang-tidy plugin", + ) + ap.add_argument( + "--config-file", + type=Path, + default=None, + help="Path to the .clang-tidy config file", + ) + ap.add_argument( + "--compdb-path", + type=Path, + required=True, + help="Path to the directory containing the cleaned compilation database", + ) + ap.add_argument( + "--werror", + action="store_true", + help="Treat warnings as errors", + ) + ap.add_argument( + "--fix", + action="store_true", + help="Apply fixes for warnings", + ) + ap.add_argument( + "--run-clang-tidy-path", + default="run-clang-tidy", + help="Path to run-clang-tidy", + ) + ap.add_argument( + "paths", + nargs="*", + help="Source paths to check", + ) + args = ap.parse_args() + + go( + args.run_clang_tidy_path, + args.plugin_path, + args.config_file, + args.compdb_path, + args.jobs, + args.paths, + args.werror, + args.fix, + ) + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py new file mode 100755 index 000000000000..8087b0bf9b6e --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Strips PCH (precompiled header) arguments from a compilation database. + +This is needed because clang-tidy cannot process PCH flags when using +nixpkgs' cc-wrapper. The cc-wrapper makes assumptions that don't hold +when the compiler is being used indirectly by clang-tidy. + +See: https://github.com/mesonbuild/meson/issues/13499 +""" + +import json +import shlex + + +def process_compdb(compdb: list[dict]) -> list[dict]: + def munch_command(args: list[str]) -> list[str]: + out = [] + eat_next = False + for i, arg in enumerate(args): + if arg in ["-fpch-preprocess", "-fpch-instantiate-templates"]: + # -fpch-preprocess as used with gcc + # -fpch-instantiate-templates as used by clang + continue + elif arg == "-include-pch" or ( + arg == "-include" + and i + 1 < len(args) + and args[i + 1] == "precompiled-headers.hh" + ): + # -include-pch some-pch (clang), or -include some-pch (gcc) + eat_next = True + continue + if not eat_next: + out.append(arg) + eat_next = False + return out + + def chomp(item: dict) -> dict: + item = item.copy() + item["command"] = shlex.join(munch_command(shlex.split(item["command"]))) + return item + + def cmdfilter(item: dict) -> bool: + file = item["file"] + # Filter out precompiled header files + if file.endswith("precompiled-headers.hh"): + return False + # Filter out Flex/Bison generated parsers (generated code) + if file.endswith("-tab.cc"): + return False + return True + + return [chomp(x) for x in compdb if cmdfilter(x)] + + +def main(): + import argparse + + ap = argparse.ArgumentParser( + description="Strip PCH arguments from compilation database" + ) + ap.add_argument("input", type=argparse.FileType("r"), help="Input json file") + ap.add_argument("output", type=argparse.FileType("w"), help="Output json file") + args = ap.parse_args() + + input_json = json.load(args.input) + json.dump(process_compdb(input_json), args.output, indent=2) + + +if __name__ == "__main__": + main() diff --git a/nix-meson-build-support/common/clang-tidy/meson.build b/nix-meson-build-support/common/clang-tidy/meson.build new file mode 100644 index 000000000000..9247c30b885a --- /dev/null +++ b/nix-meson-build-support/common/clang-tidy/meson.build @@ -0,0 +1,86 @@ +# clang-tidy integration for the Nix project +# +# This provides the 'clang-tidy' run_target for per-component builds. +# The custom plugin (nix-clang-tidy) is built as a separate component +# and loaded automatically when available. + +python = find_program('python3') +run_clang_tidy = find_program('run-clang-tidy', required : false) + +# Resolves via override_dependency (monolithic) or pkg-config (per-component Nix) +nix_clang_tidy_plugin = dependency('nix-clang-tidy', required : false) + +# Clean the compilation database by stripping PCH arguments. +# This is needed because clang-tidy cannot process PCH flags from cc-wrapper. +# See: https://github.com/mesonbuild/meson/issues/13499 +meson.add_postconf_script( + python, + meson.current_source_dir() / 'clean_compdb.py', + meson.global_build_root() / 'compile_commands.json', + meson.current_build_dir() / 'compile_commands.json', +) + +if run_clang_tidy.found() + # Pre-build generated headers/sources (.gen.hh, -tab.cc, etc.) so clang-tidy + # can run on a fresh build dir without a full `ninja` first. + # See: https://github.com/mesonbuild/meson/issues/12817 + build_gen_targets = custom_target( + 'build-gen-for-tidy', + command : [ + python, + files('build_required_targets.py'), + meson.global_build_root(), + '--stamp', '@OUTPUT@', + ], + output : 'gen-targets.stamp', + build_always_stale : true, + build_by_default : false, + ) + + # In monolithic builds the plugin .so is built from source — ensure it's + # built before we try to -load it. In per-component builds the plugin is + # already installed (buildInputs), so no extra dep is needed. + clang_tidy_deps = [ build_gen_targets ] + if is_variable('clang_tidy_plugin_sp') and clang_tidy_plugin_sp.found() + clang_tidy_deps += [ + clang_tidy_plugin_sp.get_variable('nix_clang_tidy_plugin'), + ] + endif + + run_clang_tidy_args = [ + meson.current_source_dir() / 'clang-tidy-runner.py', + '--run-clang-tidy-path', + run_clang_tidy, + '--compdb-path', + meson.current_build_dir(), + '--config-file', + meson.current_source_dir() / '.clang-tidy', + ] + + if nix_clang_tidy_plugin.found() + run_clang_tidy_args += [ + '--plugin-path', + nix_clang_tidy_plugin.get_variable('plugin_path'), + ] + endif + + run_target( + 'clang-tidy', + command : [ + python, + run_clang_tidy_args, + '--werror', + ], + depends : clang_tidy_deps, + ) + + run_target( + 'clang-tidy-fix', + command : [ + python, + run_clang_tidy_args, + '--fix', + ], + depends : clang_tidy_deps, + ) +endif diff --git a/nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc b/nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc new file mode 100644 index 000000000000..7238800b1906 --- /dev/null +++ b/nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc @@ -0,0 +1,21 @@ +#include +#include +#include + +#include "is-logic-error.hh" + +typedef void (*cxa_throw_type)(void *, std::type_info *, void (*)(void *)); + +extern "C" void __cxa_throw(void * exc, std::type_info * tinfo, void (*dest)(void *)) +{ + if (is_logic_error(tinfo)) + abort_on_exception(exc, tinfo); + + static auto * orig = (cxa_throw_type) dlsym(RTLD_NEXT, "__cxa_throw"); + if (!orig) + abort(); + + orig(exc, tinfo, dest); + + __builtin_unreachable(); +} diff --git a/nix-meson-build-support/common/cxa-throw/is-logic-error.hh b/nix-meson-build-support/common/cxa-throw/is-logic-error.hh new file mode 100644 index 000000000000..dc6285a51121 --- /dev/null +++ b/nix-meson-build-support/common/cxa-throw/is-logic-error.hh @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#ifndef CXA_THROW_ON_LOGIC_ERROR +# define CXA_THROW_ON_LOGIC_ERROR() abort() +#endif + +#include + +#ifndef __GLIBCXX__ +/* libc++abi implements the Itanium ABI RTTI classes but, unlike + libstdc++, doesn't declare them in . Their layout is + fixed by the ABI, so declare what we need. */ +namespace __cxxabiv1 { +class __class_type_info : public std::type_info +{ +public: + ~__class_type_info() override; +}; + +class __si_class_type_info : public __class_type_info +{ +public: + const __class_type_info * __base_type; +}; +} // namespace __cxxabiv1 +#endif + +static bool is_logic_error(const std::type_info * tinfo) +{ + if (*tinfo == typeid(std::logic_error)) + return true; + + auto * si = dynamic_cast(tinfo); + if (si) + return is_logic_error(si->__base_type); + + return false; +} + +static void abort_on_exception(void * exc, const std::type_info * tinfo) +{ + if (!is_logic_error(tinfo)) + return; + + char buf[512]; + snprintf( + buf, + sizeof(buf), + "Aborting on unexpected exception of type '%s', error: %s\n", + tinfo->name(), + ((std::exception *) exc)->what()); + [[maybe_unused]] auto r = write(STDERR_FILENO, buf, strlen(buf)); + + CXA_THROW_ON_LOGIC_ERROR(); +} diff --git a/nix-meson-build-support/common/cxa-throw/meson.build b/nix-meson-build-support/common/cxa-throw/meson.build new file mode 100644 index 000000000000..e9947766cd40 --- /dev/null +++ b/nix-meson-build-support/common/cxa-throw/meson.build @@ -0,0 +1,30 @@ +have_cxa_throw = false + +if host_machine.system() == 'linux' + have_cxa_throw = true + + if get_option('default_library') == 'static' + wrap_cxa_throw_args = [ '-Wl,--wrap=__cxa_throw' ] + + wrap_cxa_throw_lib = static_library( + 'wrap-cxa-throw', + 'wrap-cxa-throw.cc', + ) + + cxa_throw_dep = declare_dependency( + link_whole : wrap_cxa_throw_lib, + link_args : wrap_cxa_throw_args + [ '-Wl,-u,__wrap___cxa_throw' ], + ) + + else + interpose_cxa_throw_lib = static_library( + 'interpose-cxa-throw', + 'interpose-cxa-throw.cc', + dependencies : cxx.find_library('dl', required : false), + ) + + cxa_throw_dep = declare_dependency( + link_whole : interpose_cxa_throw_lib, + ) + endif +endif diff --git a/nix-meson-build-support/common/cxa-throw/wrap-cxa-throw.cc b/nix-meson-build-support/common/cxa-throw/wrap-cxa-throw.cc new file mode 100644 index 000000000000..f9de15533de0 --- /dev/null +++ b/nix-meson-build-support/common/cxa-throw/wrap-cxa-throw.cc @@ -0,0 +1,16 @@ +#include +#include + +#include "is-logic-error.hh" + +extern "C" void __real___cxa_throw(void *, std::type_info *, void (*)(void *)); + +extern "C" void __wrap___cxa_throw(void * exc, std::type_info * tinfo, void (*dest)(void *)) +{ + if (is_logic_error(tinfo)) + abort_on_exception(exc, tinfo); + + __real___cxa_throw(exc, tinfo, dest); + + __builtin_unreachable(); +} diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index 20dc2e3c8cf8..f7cca818057e 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -14,7 +14,7 @@ if host_machine.system() == 'cygwin' ) endif -add_project_arguments( +warning_flags = [ '-Wdeprecated-copy', '-Werror=suggest-override', '-Werror=switch', @@ -27,19 +27,40 @@ add_project_arguments( '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', + '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + '-Wno-subobject-linkage', # GCC doesn't like unity builds. + # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` + # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently + # became `.version = {16, 0}` instead of failing, breaking protocol compatibility + # in a subtle way + '-Werror=c99-designator', +] + +if meson.project_name() in [ + 'nix-util', + 'nix-store', + 'nix-util-c', + 'nix-store-c', + 'nix-expr', + 'nix-expr-c', +] + # Catch vtables with vague linkage. All vtables must have a "key" function to make + # sure they are emitted as strong symbols (as required by Itanium ABI). This is needed + # in certain cases of dynamic linking, where weak symbols are not always coalesced. Otherwise + # typeinfo symbols get duplicated and dynamic_cast/exception handling ends up broken on Darwin's + # libc++. + # Search for 'virtual void anchor' in the repo to see how this is done. + warning_flags += [ '-Werror=weak-vtables' ] +endif + +add_project_arguments( + cxx.get_supported_arguments(warning_flags), language : 'cpp', ) # GCC doesn't benefit much from precompiled headers. do_pch = cxx.get_id() == 'clang' -if cxx.get_id() == 'gcc' - add_project_arguments( - '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. - language : 'cpp', - ) -endif - # This is a clang-only option for improving build times. # It forces the instantiation of templates in the PCH itself and # not every translation unit it's included in. @@ -49,11 +70,6 @@ endif # instantiations in libutil and libstore. if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') - # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` - # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently - # became `.version = {16, 0}` instead of failing, breaking protocol compatibility - # in a subtle way - add_project_arguments('-Werror=c99-designator', language : 'cpp') endif # Detect if we're using libstdc++ (GCC's standard library) @@ -76,8 +92,18 @@ if is_using_libstdcxx add_project_arguments('-D_GLIBCXX_USE_TBB_PAR_BACKEND=0', language : 'cpp') endif -# Darwin ld doesn't like "X.Y.ZpreABCD+W" -nix_soversion = meson.project_version().split('+')[0].split('pre')[0] +cxx = meson.get_compiler('cpp') + +# Clang does not support prelinking on static builds +if cxx.get_id() == 'clang' and get_option('default_library') == 'static' + prelink = false +else + prelink = true +endif subdir('assert-fail') subdir('asan-options') +subdir('cxa-throw') +if not meson.is_subproject() + subdir('clang-tidy') +endif diff --git a/nix-meson-build-support/default-system-cpu/meson.build b/nix-meson-build-support/default-system-cpu/meson.build index f63b07975b6e..3e872578efca 100644 --- a/nix-meson-build-support/default-system-cpu/meson.build +++ b/nix-meson-build-support/default-system-cpu/meson.build @@ -14,6 +14,6 @@ if (host_machine.cpu_family() in [ 'ppc64', 'ppc' ]) and host_machine.endian() = nix_system_cpu += 'le' elif host_machine.cpu_family() in [ 'mips64', 'mips' ] and host_machine.endian() == 'little' nix_system_cpu += 'el' -elif host_machine.cpu_family() == 'arm' +elif host_machine.cpu_family() in [ 'arm', 'arm64' ] nix_system_cpu = host_machine.cpu() endif diff --git a/nix-meson-build-support/export/meson.build b/nix-meson-build-support/export/meson.build index 62a27bd48c37..d5cb39661d06 100644 --- a/nix-meson-build-support/export/meson.build +++ b/nix-meson-build-support/export/meson.build @@ -11,12 +11,34 @@ endforeach requires_public += deps_public extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {}) +fs = import('fs') +plugin_c_api_enabled = get_variable('plugin_c_api_enabled', false) + +if plugin_c_api_enabled + this_libraries = get_variable('this_libraries') + this_library = this_libraries.get_shared_lib() + this_static_library = this_libraries.get_static_lib() +else + this_library = get_variable('this_library') +endif extra_cflags = [] if not meson.project_name().endswith('-c') extra_cflags += [ '-std=c++23' ] endif +whole_archive_dep_name = meson.project_name() + '-whole-archive' +installed_whole_archive_link_arg = '' +if plugin_c_api_enabled and host_machine.system() == 'darwin' + installed_whole_archive_link_arg = '-Wl,-force_load,${libdir}/' + fs.name( + this_static_library.full_path(), + ) +elif plugin_c_api_enabled and not host_machine.system().startswith('windows') + installed_whole_archive_link_arg = '-Wl,--whole-archive,${libdir}/' + fs.name( + this_static_library.full_path(), + ) + ',--no-whole-archive' +endif + import('pkgconfig').generate( this_library, filebase : meson.project_name(), @@ -29,6 +51,19 @@ import('pkgconfig').generate( variables : extra_pkg_config_variables, ) +if installed_whole_archive_link_arg != '' + import('pkgconfig').generate( + filebase : whole_archive_dep_name, + name : 'Nix whole-archive static dependency', + description : 'Force-load static Nix C API archive into the host executable', + extra_cflags : extra_cflags, + requires : requires_public, + requires_private : requires_private, + libraries : [ installed_whole_archive_link_arg ], + variables : extra_pkg_config_variables, + ) +endif + meson.override_dependency( meson.project_name(), declare_dependency( @@ -39,3 +74,16 @@ meson.override_dependency( variables : extra_pkg_config_variables, ), ) + +if installed_whole_archive_link_arg != '' + meson.override_dependency( + whole_archive_dep_name, + declare_dependency( + include_directories : include_dirs, + link_whole : [ this_static_library ], + compile_args : [ '-std=c++23' ], + dependencies : deps_public_subproject + deps_public, + variables : extra_pkg_config_variables, + ), + ) +endif diff --git a/packaging/components.nix b/packaging/components.nix index b74484b6e9a2..50629ef1bd73 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -27,7 +27,7 @@ let pkg-config ; - baseVersion = lib.fileContents ../.version; + baseVersion = lib.fileContents ../.version-determinate; versionSuffix = lib.optionalString (!officialRelease) "pre"; @@ -45,21 +45,15 @@ let filesetToSource = lib.fileset.toSource; /** - Given a set of layers, create a mkDerivation-like function + Given a stdenv and set of layers, create a mkDerivation-like function */ - mkPackageBuilder = - exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + mkPackageBuilderFor = + stdenv': exts: userFn: + stdenv'.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + + mkPackageBuilder = mkPackageBuilderFor stdenv; setVersionLayer = finalAttrs: prevAttrs: { - preConfigure = - prevAttrs.preConfigure or "" - + - # Update the repo-global .version file. - # Symlink ./.version points there, but by default only workDir is writable. - '' - chmod u+w ./.version - echo ${finalAttrs.version} > ./.version - ''; }; localSourceLayer = @@ -136,8 +130,6 @@ let !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin) # build failure && !stdenv.hostPlatform.isStatic - # LTO breaks exception handling on x86-64-darwin. - && stdenv.system != "x86_64-darwin" ) '' case "$mesonBuildType" in @@ -150,6 +142,13 @@ let ninja ] ++ prevAttrs.nativeBuildInputs or [ ]; + mesonFlags = + prevAttrs.mesonFlags or [ ] + ++ (lib.optionals scope.withUnityBuild [ + "-Dunity=on" + "-Dunity_size=8192" + "-Db_pch=false" + ]); mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ "--print-errorlogs" ]; @@ -209,8 +208,13 @@ let enableSanitizersLayer = finalAttrs: prevAttrs: let - sanitizers = lib.optional scope.withASan "address" ++ lib.optional scope.withUBSan "undefined"; + sanitizers = + lib.optional scope.withASan "address" + ++ lib.optional scope.withUBSan "undefined" + ++ lib.optional scope.withTSan "thread"; in + # Thread sanitizer can't be used with ASan or UBSan + assert scope.withTSan -> !(scope.withASan || scope.withUBSan); { mesonFlags = (prevAttrs.mesonFlags or [ ]) @@ -225,6 +229,30 @@ let ); }; + enableClangTidyLayer = + finalAttrs: prevAttrs: + lib.optionalAttrs scope.withClangTidy { + nativeBuildInputs = (prevAttrs.nativeBuildInputs or [ ]) ++ [ + pkgs.buildPackages.llvmPackages.clang-tools # provides run-clang-tidy + ]; + + buildInputs = (prevAttrs.buildInputs or [ ]) ++ [ + scope.nix-clang-tidy-plugin # provides nix-clang-tidy.pc with plugin_path + ]; + + # Use debug build for faster compilation (no optimizations) + mesonBuildType = "debug"; + + # Skip tests - we only care about clang-tidy results + doCheck = false; + + # Run clang-tidy after the normal build + postBuild = (prevAttrs.postBuild or "") + '' + echo "Running clang-tidy on ${finalAttrs.pname}..." + ninja clang-tidy + ''; + }; + nixDefaultsLayer = finalAttrs: prevAttrs: { strictDeps = prevAttrs.strictDeps or true; enableParallelBuilding = true; @@ -277,6 +305,26 @@ in */ withUBSan = false; + /** + Whether meson components are built with [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html). + */ + withTSan = false; + + /** + Whether meson components are checked with [clang-tidy](https://clang.llvm.org/extra/clang-tidy/). + */ + withClangTidy = false; + + /** + Whether to use [unity builds](https://mesonbuild.com/Unity-builds.html#unity-builds). + */ + withUnityBuild = true; + + /** + Whether to embed the public C API into nix-cli so plugins can resolve those symbols from the executable. + */ + withPluginCAPI = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + /** A user-provided extension function to apply to each component derivation. */ @@ -375,6 +423,7 @@ in mesonBuildLayer fixupStaticLayer enableSanitizersLayer + enableClangTidyLayer scope.mesonComponentOverrides ]; mkMesonLibrary = mkPackageBuilder [ @@ -387,6 +436,7 @@ in mesonLibraryLayer fixupStaticLayer enableSanitizersLayer + enableClangTidyLayer scope.mesonComponentOverrides ]; @@ -423,7 +473,9 @@ in /** The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does. */ - nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-cli = callPackage ../src/nix/package.nix { + version = fineVersion; + }; nix-functional-tests = callPackage ../tests/functional/package.nix { version = fineVersion; @@ -456,7 +508,21 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; + # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, + # so it must be built with a clang stdenv for ABI compatibility with the + # clang-tidy binary from the same llvmPackages set, regardless of the + # scope's stdenv (which may be GCC). + nix-clang-tidy-plugin = callPackage ../src/clang-tidy-plugin/package.nix { + llvmPackages = pkgs.buildPackages.llvmPackages; + mkMesonDerivation = mkPackageBuilderFor pkgs.buildPackages.llvmPackages.stdenv [ + nixDefaultsLayer + scope.sourceLayer + setVersionLayer + mesonLayer + fixupStaticLayer + scope.mesonComponentOverrides + ]; + }; /** Combined package that has the CLI, libraries, and (assuming non-cross, no overrides) it requires that all tests succeed. diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 5338f70e57b3..480c4079e954 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,38 +16,177 @@ in scope: { inherit stdenv; + mimalloc = + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then + pkgs.mimalloc + else + pkgs.mimalloc.overrideAttrs rec { + version = "3.3.2"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; + }; + }; + boehmgc = (pkgs.boehmgc.override { enableLargeConfig = true; + inherit stdenv; }).overrideAttrs (attrs: { - # Increase the initial mark stack size to avoid stack - # overflows, since these inhibit parallel marking (see - # GC_mark_some()). To check whether the mark stack is too - # small, run Nix with GC_PRINT_STATS=1 and look for messages - # such as `Mark stack overflow`, `No room to copy back mark - # stack`, and `Grew mark stack to ... frames`. - NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576"; + # Reduce contention on the GC allocation lock during parallel + # evaluation by handing out multiple heap blocks worth of + # objects per lock acquisition in GC_generic_malloc_many(). + # The default batch size is set via GC_MANY_BLOCKS_DEFAULT + # below and can be overridden at runtime through the + # GC_MALLOC_MANY_BLOCKS environment variable. + patches = (attrs.patches or [ ]) ++ [ ./patches/boehmgc-batch-malloc-many.patch ]; + + env = (attrs.env or { }) // { + # Increase the initial mark stack size to avoid stack + # overflows, since these inhibit parallel marking (see + # GC_mark_some()). To check whether the mark stack is too + # small, run Nix with GC_PRINT_STATS=1 and look for messages + # such as `Mark stack overflow`, `No room to copy back mark + # stack`, and `Grew mark stack to ... frames`. + NIX_CFLAGS_COMPILE = toString ( + [ + "-DINITIAL_MARK_STACK_SIZE=1048576" + "-DGC_MANY_BLOCKS_DEFAULT=64" + ] + # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. + # Force this to be built with compiler-rt & libunwind over libgcc_eh works. + # Issue: https://github.com/NixOS/nixpkgs/issues/177129 + ++ + lib.optionals + ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) + [ + "-rtlib=compiler-rt" + "-unwindlib=libunwind" + ] + ); + }; + + buildInputs = + (attrs.buildInputs or [ ]) + ++ lib.optional ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) pkgs.llvmPackages.libunwind; }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + lowdown = + if lib.versionAtLeast pkgs.lowdown.version "2.0.2" then + pkgs.lowdown + else + pkgs.lowdown.overrideAttrs (prevAttrs: rec { + version = "2.0.2"; + src = pkgs.fetchurl { + url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz"; + hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA=="; + }; + nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ]; + postInstall = + lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ] + (prevAttrs.postInstall or ""); + }); - libblake3 = pkgs.libblake3.override { - useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + # libpsl uses a data file needed at runtime, not useful for nix. + pslSupport = !stdenv.hostPlatform.isStatic; + idnSupport = !stdenv.hostPlatform.isStatic; }; + libblake3 = + (pkgs.libblake3.override { + inherit stdenv; + # Nixpkgs disables tbb on static + useTBB = + !( + stdenv.hostPlatform.isWindows + || stdenv.hostPlatform.isStatic + # Some tbb tests fail with libc++. + || (stdenv.cc.libcxx != null && stdenv.cc.libcxx.isLLVM) + ); + }) + # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. + # Force this to be built with compiler-rt & libunwind over libgcc_eh works. + # Issue: https://github.com/NixOS/nixpkgs/issues/177129 + .overrideAttrs + ( + attrs: + lib.optionalAttrs + ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) + { + NIX_CFLAGS_COMPILE = [ + "-rtlib=compiler-rt" + "-unwindlib=libunwind" + ]; + + buildInputs = [ + pkgs.llvmPackages.libunwind + ]; + } + ); + + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + + libgit2 = + ( + if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.4"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-ZKUiz3pdFE2SKxh53X2oyr7hs32Njj5YVA0OXDXz7h0="; + }; + } + ).overrideAttrs + (old: { + separateDebugInfo = true; + + patches = old.patches or [ ] ++ [ + # Fix a use-after-free crash when `git_thread_create` fails during + # pack building (e.g. with EAGAIN under thread pressure), leaving + # orphaned delta-search worker threads running while the + # packbuilder is freed. + ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch + ]; + }); + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { @@ -57,12 +196,32 @@ scope: { "--with-coroutine" "--with-iostreams" "--with-url" + "--with-thread" + ]; + patches = [ + ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch ]; enableIcu = false; + inherit stdenv; }).overrideAttrs (old: { # Need to remove `--with-*` to use `--with-libraries=...` buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); + + wasmtime = pkgs.callPackage ./wasmtime.nix { }; + + sentry-native = (pkgs.callPackage ./sentry-native.nix { }).override { + # Avoid having two curls in our closure. + inherit (scope) curl; + }; + + libmicrohttpd = pkgs.libmicrohttpd.overrideDerivation (old: { + # Don't pull in gnutls since it's pretty big and we don't need it. + configureFlags = old.configureFlags or [ ] ++ [ "--without-gnutls" ]; + + # Required for configuration detection for getsockname (for automatic port allocation for `nix serve`) + __darwinAllowLocalNetworking = true; + }); } diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 72d517d876a8..ad634d475d19 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -110,245 +110,242 @@ let in -{ pkgs }: - -# TODO: don't use nix-util for this? -pkgs.nixComponents2.nix-util.overrideAttrs ( - finalAttrs: prevAttrs: - - let - stdenv = pkgs.nixDependencies2.stdenv; - buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - modular = devFlake.getSystem stdenv.buildPlatform.system; - transformFlag = - prefix: flag: - assert builtins.isString flag; - let - rest = builtins.substring 2 (builtins.stringLength flag) flag; - in - "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; - ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; - - availableComponents = lib.filterAttrs ( - k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v - ) allComponents; - - activeComponents = buildInputsClosureCond isInternal ( - lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) - ); +{ pkgs, nixComponents }: - allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; - internalDrvs = byDrvPath ( - # Drop the attr names (not present in buildInputs anyway) - lib.attrValues availableComponents - ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( - lib.attrValues availableComponents - ) - ); +nixComponents.callPackage ( + { stdenv }: + (stdenv.mkDerivation ( + finalAttrs: - isInternal = - dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; + let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = + prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + + availableComponents = lib.filterAttrs ( + k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v + ) allComponents; + + activeComponents = buildInputsClosureCond isInternal ( + lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) + ); - activeComponentNames = lib.listToAttrs ( - map (c: { - name = c.pname or c.name; - value = null; - }) activeComponents - ); + allComponents = lib.filterAttrs (k: v: lib.isDerivation v) nixComponents; + internalDrvs = byDrvPath ( + # Drop the attr names (not present in buildInputs anyway) + lib.attrValues availableComponents + ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( + lib.attrValues availableComponents + ) + ); - isActiveComponent = name: activeComponentNames ? ${name}; + isInternal = + dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; - in - { - pname = "shell-for-nix"; + activeComponentNames = lib.listToAttrs ( + map (c: { + name = c.pname or c.name; + value = null; + }) activeComponents + ); - passthru = { - inherit activeComponents; + isActiveComponent = name: activeComponentNames ? ${name}; - # We use this attribute to store non-derivation values like functions and - # perhaps other things that are primarily for overriding and not the shell. - config = { - # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + in + { + pname = "shell-for-nix"; + + passthru = { + inherit activeComponents; + + # We use this attribute to store non-derivation values like functions and + # perhaps other things that are primarily for overriding and not the shell. + config = { + # Default getComponents + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); + }; + + /** + Produce a devShell for a given set of nix components + + Example: + + ```nix + shell.withActiveComponents (c: { + inherit (c) nix-util; + }) + ``` + */ + withActiveComponents = + f2: + finalAttrs.finalPackage.overrideAttrs ( + finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru // { + config = prevAttrs.passthru.config // { + getComponents = f2; + }; + }; + } ); - }; - /** - Produce a devShell for a given set of nix components - - Example: - - ```nix - shell.withActiveComponents (c: { - inherit (c) nix-util; - }) - ``` - */ - withActiveComponents = - f2: - finalAttrs.finalPackage.overrideAttrs ( - finalAttrs: prevAttrs: { - passthru = prevAttrs.passthru // { - config = prevAttrs.passthru.config // { - getComponents = f2; - }; - }; - } + small = finalAttrs.finalPackage.withActiveComponents ( + c: + lib.intersectAttrs (lib.genAttrs [ + "nix-cli" + "nix-util-tests" + "nix-store-tests" + "nix-expr-tests" + "nix-fetchers-tests" + "nix-flake-tests" + "nix-functional-tests" + ] (_: null)) c ); + }; - small = finalAttrs.finalPackage.withActiveComponents ( - c: - lib.intersectAttrs (lib.genAttrs [ - "nix-cli" - "nix-util-tests" - "nix-store-tests" - "nix-expr-tests" - "nix-fetchers-tests" - "nix-flake-tests" - "nix-functional-tests" - "nix-perl-bindings" - ] (_: null)) c - ); - }; - - # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version; - name = finalAttrs.pname; - - installFlags = "sysconfdir=$(out)/etc"; - shellHook = '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - - # Make the default phases do the right thing. - # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. - # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. - - cdToBuildDir() { - if [[ ! -e build.ninja ]]; then - cd build - fi - } - - configurePhase() { - mesonConfigurePhase - } - - buildPhase() { - cdToBuildDir - ninjaBuildPhase - } - - checkPhase() { - cdToBuildDir - mesonCheckPhase - } - - installPhase() { - cdToBuildDir - ninjaInstallPhase + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ../.version-determinate; + name = finalAttrs.pname; + + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + + # Make the default phases do the right thing. + # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. + # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. + + cdToBuildDir() { + if [[ ! -e build.ninja ]]; then + cd build + fi + } + + configurePhase() { + mesonConfigurePhase + } + + buildPhase() { + cdToBuildDir + ninjaBuildPhase + } + + checkPhase() { + cdToBuildDir + mesonCheckPhase + } + + installPhase() { + cdToBuildDir + ninjaInstallPhase + } + ''; + + # We use this shell with the local checkout, not unpackPhase. + src = null; + + # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html + # Remove when gdb fix is rolled out everywhere. + separateDebugInfo = false; + + mesonBuildType = "debugoptimized"; + + env = { + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" + modular.pre-commit.settings.rawConfig + }"; } - ''; - - # We use this shell with the local checkout, not unpackPhase. - src = null; - - # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html - # Remove when gdb fix is rolled out everywhere. - separateDebugInfo = false; - - mesonBuildType = "debugoptimized"; + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + CC_LD = "mold"; + CXX_LD = "mold"; + }; - env = { - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" - modular.pre-commit.settings.rawConfig - }"; - } - // lib.optionalAttrs stdenv.hostPlatform.isLinux { - CC_LD = "mold"; - CXX_LD = "mold"; - }; - - dontUseCmakeConfigure = true; - - mesonFlags = [ - (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) - ] - ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags) - ) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags); - - nativeBuildInputs = - let - inputs = - dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap ( - # Nix manual has a build-time dependency on nix, but we - # don't want to do a native build just to enter the cross - # dev shell. - # - # TODO: think of a more principled fix for this. - c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs - ) activeComponents + dontUseCmakeConfigure = true; + + mesonFlags = [ + (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) + ] + ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) + ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) + ++ map (transformFlag "libutil-c") (ignoreCrossFile nixComponents.nix-util-c.mesonFlags) + ++ map (transformFlag "libstore-c") (ignoreCrossFile nixComponents.nix-store-c.mesonFlags) + ++ map (transformFlag "libfetchers-c") (ignoreCrossFile nixComponents.nix-fetchers-c.mesonFlags) + ++ map (transformFlag "libexpr-c") (ignoreCrossFile nixComponents.nix-expr-c.mesonFlags) + ++ map (transformFlag "libflake-c") (ignoreCrossFile nixComponents.nix-flake-c.mesonFlags) + ++ map (transformFlag "libmain-c") (ignoreCrossFile nixComponents.nix-main-c.mesonFlags) + ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); + + nativeBuildInputs = + let + inputs = + dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap ( + # Nix manual has a build-time dependency on nix, but we + # don't want to do a native build just to enter the cross + # dev shell. + # + # TODO: think of a more principled fix for this. + c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs + ) activeComponents + ) ) - ) - ++ lib.optional ( - !buildCanExecuteHost - # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 - && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) - && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages - && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) - ) pkgs.buildPackages.mesonEmulatorHook - ++ [ - pkgs.buildPackages.gnused - modular.pre-commit.settings.package - (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) - pkgs.buildPackages.nixfmt-rfc-style - pkgs.buildPackages.shellcheck - pkgs.buildPackages.include-what-you-use - ] - ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( - lib.hiPrio pkgs.buildPackages.clang-tools - ) - ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; - in - # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections - # from making its way into NIX_CFLAGS_COMPILE. - lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; - - propagatedNativeBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents - ) - ); + ++ lib.optional ( + !buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) + ) pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.gnused + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) + pkgs.buildPackages.nixfmt + pkgs.buildPackages.shellcheck + pkgs.buildPackages.include-what-you-use + ] + ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( + lib.hiPrio pkgs.buildPackages.clang-tools + ) + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold; + in + # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections + # from making its way into NIX_CFLAGS_COMPILE. + lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; + + propagatedNativeBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + ) + ); - buildInputs = - # TODO change Nixpkgs to mark gbenchmark as building on Windows - lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark - ++ dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + buildInputs = + # TODO change Nixpkgs to mark gbenchmark as building on Windows + lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark + ++ dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) + ); - propagatedBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) - ); - } -) + propagatedBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) + ); + } + )) +) { } diff --git a/packaging/everything.nix b/packaging/everything.nix index 4d8f94f4b9ab..fdb72663fcca 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,11 +41,13 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, + + curl, + boehmgc, + sentry-native, }: let @@ -65,19 +67,10 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { - name = "nix-${nix-cli.version}-devdoc"; + name = "determinate-nix-${nix-cli.version}-devdoc"; paths = [ nix-internal-api-docs nix-external-api-docs @@ -86,7 +79,7 @@ let in stdenv.mkDerivation (finalAttrs: { - pname = "nix"; + pname = "determinate-nix"; version = nix-cli.version; /** @@ -104,6 +97,7 @@ stdenv.mkDerivation (finalAttrs: { "dev" "doc" "man" + "debug" ]; /** @@ -116,7 +110,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ @@ -141,11 +135,9 @@ stdenv.mkDerivation (finalAttrs: { nix-functional-tests ] ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings + nix-util-tests.tests.run-without-new-syscalls ]; nativeBuildInputs = [ @@ -155,9 +147,18 @@ stdenv.mkDerivation (finalAttrs: { installPhase = let devPaths = lib.mapAttrsToList (_k: lib.getDev) finalAttrs.finalPackage.libs; + debugPaths = lib.map (lib.getOutput "debug") ( + lib.attrValues finalAttrs.finalPackage.libs + ++ [ + nix-cli + curl + boehmgc + ] + ++ lib.optional (stdenv.hostPlatform.isLinux && !stdenv.hostPlatform.isStatic) sentry-native + ); in '' - mkdir -p $out $dev/nix-support + mkdir -p $out $dev/nix-support $debug/lib/debug # Custom files echo $libs >> $dev/nix-support/propagated-build-inputs @@ -170,6 +171,12 @@ stdenv.mkDerivation (finalAttrs: { lndir $lib $dev done + for d in ${lib.escapeShellArgs debugPaths}; do + if [[ -d $d/lib/debug ]]; then + lndir $d/lib/debug $debug/lib/debug + fi + done + # Forwarded outputs ln -sT ${nix-manual} $doc ln -sT ${nix-manual.man} $man diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9b9b33d6798b..5600d394ebb5 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -64,9 +63,7 @@ let "nix-cli" "nix-functional-tests" "nix-json-schema-checks" - ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" + "nix-clang-tidy-plugin" ] ++ lib.optionals enableDocs [ "nix-manual" @@ -84,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -126,95 +122,25 @@ rec { system: self.devShells.${system}.default.inputDerivation )) [ "i686-linux" ]; - buildStatic = forAllPackages ( - pkgName: - lib.genAttrs linux64BitSystems ( - system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName} - ) - ); - - buildCross = forAllPackages ( - pkgName: - # Hack to avoid non-evaling package - ( - if pkgName == "nix-functional-tests" then - lib.flip builtins.removeAttrs [ "x86_64-w64-mingw32" ] - else if pkgName == "nix-nswrapper" then - lib.filterAttrs ( - crossSystem: _do_not_touch: nixpkgsFor.x86_64-linux.cross.${crossSystem}.stdenv.hostPlatform.isLinux - ) - else - lib.id - ) - ( - forAllCrossSystems ( - crossSystem: - lib.genAttrs [ "x86_64-linux" ] ( - system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName} - ) - ) - ) - ); - - # Builds with sanitizers already have GC disabled, so this buildNoGc can just - # point to buildWithSanitizers in order to reduce the load on hydra. - buildNoGc = buildWithSanitizers; - - buildWithSanitizers = - let - components = forAllSystems ( - system: - let - pkgs = nixpkgsFor.${system}.native; - in - pkgs.nixComponents2.overrideScope ( - self: super: { - # Boost coroutines fail with ASAN on darwin. - withASan = !pkgs.stdenv.buildPlatform.isDarwin; - withUBSan = true; - nix-expr = super.nix-expr.override { enableGC = false; }; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = null; - } - ) - ); - in - forAllPackages ( - pkgName: - lib.filterAttrs ( - system: _do_not_touch: - pkgName == "nix-nswrapper" -> nixpkgsFor.${system}.native.stdenv.hostPlatform.isLinux - ) (forAllSystems (system: components.${system}.${pkgName})) - ); - - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = + # Static analysis with clang-tidy + clangTidy = lib.genAttrs linux64BitSystems ( + system: let - components = forAllSystems ( - system: - nixpkgsFor.${system}.native.nixComponents2.overrideScope ( - self: super: { - nix-cmd = super.nix-cmd.override { - enableMarkdown = false; - readlineFlavor = "readline"; - }; - } - ) + pkgs = nixpkgsFor.${system}.nativeForStdenv.clangStdenv; + tidyScope = pkgs.nixComponents2.overrideScope ( + self: super: { + withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; + # nix-everything is built via callPackage (not the layer system), so + # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here + # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. + nix-everything = super.nix-everything.overrideAttrs { doCheck = false; }; + } ); in - forAllPackages ( - pkgName: - lib.filterAttrs ( - system: _do_not_touch: - pkgName == "nix-nswrapper" -> nixpkgsFor.${system}.native.stdenv.hostPlatform.isLinux - ) (forAllSystems (system: components.${system}.${pkgName})) - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); + tidyScope.nix-everything + ); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -223,30 +149,6 @@ rec { system: nixpkgsFor.${system}.native.callPackage ./binary-tarball.nix { } ); - binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] ( - system: - forAllCrossSystems ( - crossSystem: nixpkgsFor.${system}.cross.${crossSystem}.callPackage ./binary-tarball.nix { } - ) - ); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; - installerScriptForGHA = forAllSystems ( system: nixpkgsFor.${system}.native.callPackage ./installer { @@ -254,6 +156,31 @@ rec { } ); + # `NixOS/nix-installer` with this revision's Nix closure embedded. + rustInstaller = + lib.genAttrs + ( + linux64BitSystems + ++ [ + "aarch64-darwin" + ] + ) + ( + system: + let + pkgs = nixpkgsFor.${system}.native; + # Embed the native (glibc) Nix even though the Linux installer + # binary is static/musl. + tarball = pkgs.callPackage ./rust-installer/tarball.nix { + nix = pkgs.nixComponents2.nix-everything; + }; + builder = if pkgs.stdenv.hostPlatform.isLinux then pkgs.pkgsStatic else pkgs; + in + builder.callPackage ./rust-installer { + inherit tarball; + } + ); + # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); @@ -314,6 +241,26 @@ rec { pkgs = nixpkgsFor.${system}.native; } ); + + nixpkgsLibTestsLazy = forAllSystems ( + system: + lib.overrideDerivation + (import (nixpkgs + "/lib/tests/test-with-nix.nix") { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix-cli; + pkgs = nixpkgsFor.${system}.native; + }) + (_: { + "NIX_CONFIG" = "lazy-trees = true"; + }) + ); + + filetransfer-retry-backoff = forAllSystems ( + system: + nixpkgsFor.${system}.native.callPackage ../tests/filetransfer-retry-backoff { + nix = nixpkgsFor.${system}.native.nixComponents2.nix-cli; + } + ); }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { @@ -328,17 +275,12 @@ rec { in pkgs.runCommand "install-tests" { againstSelf = testNixVersions pkgs pkgs.nix; - againstCurrentLatest = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null; + #againstCurrentLatest = + # # FIXME: temporarily disable this on macOS because of #3605. + # if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null; # Disabled because the latest stable version doesn't handle # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work # againstLatestStable = testNixVersions pkgs pkgs.nixStable; } "touch $out" ); - - installerTests = import ../tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; } diff --git a/packaging/installer/default.nix b/packaging/installer/default.nix index e171f36f99f7..a8e344b496c8 100644 --- a/packaging/installer/default.nix +++ b/packaging/installer/default.nix @@ -32,7 +32,7 @@ runCommand "installer-script" in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballHash_${system}@' $(nix hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) tarballs diff --git a/packaging/installer/install.in b/packaging/installer/install.in index b4e808d8e941..af61a433ddf5 100755 --- a/packaging/installer/install.in +++ b/packaging/installer/install.in @@ -65,6 +65,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-darwin@ system=aarch64-darwin ;; + FreeBSD.amd64|FreeBSD.x86_64) + hash=@tarballHash_x86_64-freebsd@ + path=@tarballPath_x86_64-freebsd@ + system=x86_64-freebsd + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch new file mode 100644 index 000000000000..7ec13724c40b --- /dev/null +++ b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch @@ -0,0 +1,102 @@ +From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 +From: Sergei Zimmerman +Date: Tue, 21 Jul 2026 21:15:51 +0000 +Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind + +Unwound fibers would see std::uncaught_exceptions() == 0, while a +forced_unwind exception is in "flight". This goes against the contract +of std::uncaught_exceptions() that scope guards rely upon. Failing +to report the correct number of uncaught exceptions (especially +misreporting zero) will lead to scope guards to misbehave badly and skip +running cleanup code which branches on whether the destructor is called +during stack unwinding or not. + +This is because the "throw" would happen before the destructor is run on +the fiber stack being switched to, but the increment would be clobbered +by the destructor of manage_exception_state. + +I'm not sure what the contract of run ontop_fcontext is wrt to whether +the the caller provided function can throw or not, but in my best +understanding the forced_unwind mechanism is mostly internal and so is +throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept +the catch block scoped to detail::forced_unwind. +--- + include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- + test/test_fiber.cpp | 24 +++++++++++++++ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp +index 543ba6c..38476c9 100644 +--- a/boost/context/fiber_fcontext.hpp ++++ b/boost/context/fiber_fcontext.hpp +@@ -70,7 +70,9 @@ namespace context { + namespace detail { + + // manage_exception_state is a dummy struct unless we have specific support +-struct manage_exception_state {}; ++struct manage_exception_state { ++ void from_forced_unwind() noexcept {} ++}; + + } // namespace detail + } // namespace context +@@ -90,6 +92,11 @@ public: + manage_exception_state() { + exception_state_ = *__cxa_get_globals(); + } ++ // Hack to account for the forced_unwind exception thrown in fiber_unwind ++ // that's run ontop before the destructor. ++ void from_forced_unwind() noexcept { ++ exception_state_.uncaughtExceptions += 1; ++ } + ~manage_exception_state() { + *__cxa_get_globals() = exception_state_; + } +@@ -376,13 +383,18 @@ public: + BOOST_ASSERT( nullptr != fctx_); + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); +- return { detail::jump_fcontext( ++ try { ++ return { detail::jump_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- nullptr).fctx }; ++ nullptr).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + template< typename Fn > +@@ -391,14 +403,19 @@ public: + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); + auto p = std::forward< Fn >( fn); +- return { detail::ontop_fcontext( ++ try { ++ return { detail::ontop_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- & p, +- detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ & p, ++ detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + explicit operator bool() const noexcept { diff --git a/packaging/patches/boehmgc-batch-malloc-many.patch b/packaging/patches/boehmgc-batch-malloc-many.patch new file mode 100644 index 000000000000..2e40f85796b9 --- /dev/null +++ b/packaging/patches/boehmgc-batch-malloc-many.patch @@ -0,0 +1,154 @@ +Reduce contention on the GC allocation lock: make GC_generic_malloc_many() +hand out up to GC_many_blocks heap blocks worth of objects per acquisition +of the allocation lock, instead of exactly one. This matters for +parallel evaluation in Nix, where all evaluator threads replenish their +thread-local free lists (Values, Envs, Bindings) through this function +and otherwise serialize on the allocation lock. + +The batch size is settable via the GC_MALLOC_MANY_BLOCKS environment +variable (1..64); the compile-time default is GC_MANY_BLOCKS_DEFAULT +(1 = upstream behavior, overridden in packaging/dependencies.nix). + +diff -ur a/include/private/gc_priv.h b/include/private/gc_priv.h +--- a/include/private/gc_priv.h 2026-07-03 12:56:51.388341746 +0200 ++++ b/include/private/gc_priv.h 2026-07-09 20:00:13.787169270 +0200 +@@ -1767,6 +1767,17 @@ + + GC_EXTERN unsigned GC_n_kinds; + ++/* Number of heap blocks worth of objects to hand out per allocation */ ++/* lock acquisition in GC_generic_malloc_many(). Larger values reduce */ ++/* contention on the allocation lock when many threads allocate small */ ++/* objects heavily. Settable at runtime via the GC_MALLOC_MANY_BLOCKS */ ++/* environment variable (clamped to [1, GC_MANY_BLOCKS_MAX]). */ ++#ifndef GC_MANY_BLOCKS_DEFAULT ++# define GC_MANY_BLOCKS_DEFAULT 1 ++#endif ++#define GC_MANY_BLOCKS_MAX 64 ++GC_EXTERN int GC_many_blocks; ++ + GC_EXTERN size_t GC_page_size; + + /* Round up allocation size to a multiple of a page size. */ +diff -ur a/mallocx.c b/mallocx.c +--- a/mallocx.c 2026-07-03 12:56:51.389341768 +0200 ++++ b/mallocx.c 2026-07-09 20:00:13.787169270 +0200 +@@ -300,6 +300,9 @@ + /* since the collector would not retain the entire list if it were */ + /* invoked just as we were returning. */ + /* Note that the client should usually clear the link field. */ ++/* See the description in gc_priv.h. */ ++GC_INNER int GC_many_blocks = GC_MANY_BLOCKS_DEFAULT; ++ + GC_API void GC_CALL GC_generic_malloc_many(size_t lb, int k, void **result) + { + void *op; +@@ -433,7 +436,7 @@ + my_bytes_allocd = 0; + for (p = op; p != 0; p = obj_link(p)) { + my_bytes_allocd += lb; +- if ((word)my_bytes_allocd >= HBLKSIZE) { ++ if ((word)my_bytes_allocd >= (word)GC_many_blocks * HBLKSIZE) { + *opp = obj_link(p); + obj_link(p) = 0; + break; +@@ -442,12 +445,24 @@ + GC_bytes_allocd += my_bytes_allocd; + goto out; + } +- /* Next try to allocate a new block worth of objects of this size. */ ++ /* Next try to allocate new blocks worth of objects of this size. */ ++ /* Up to GC_many_blocks heap blocks are allocated per acquisition */ ++ /* of the allocation lock, to reduce contention on it. */ + { +- struct hblk *h = GC_allochblk(lb, k, 0); +- if (h /* != NULL */) { /* CPPCHECK */ +- if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h)); ++ struct hblk *hbs[GC_MANY_BLOCKS_MAX]; ++ int nblocks = GC_many_blocks; ++ int i; ++ ++ if (nblocks < 1) nblocks = 1; ++ if (nblocks > GC_MANY_BLOCKS_MAX) nblocks = GC_MANY_BLOCKS_MAX; ++ for (i = 0; i < nblocks; i++) { ++ hbs[i] = GC_allochblk(lb, k, 0); ++ if (NULL == hbs[i]) break; ++ if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(hbs[i])); + GC_bytes_allocd += HBLKSIZE - HBLKSIZE % lb; ++ } ++ nblocks = i; ++ if (nblocks > 0) { + # ifdef PARALLEL_MARK + if (GC_parallel) { + GC_acquire_mark_lock(); +@@ -455,8 +470,10 @@ + UNLOCK(); + GC_release_mark_lock(); + +- op = GC_build_fl(h, lw, +- (ok -> ok_init || GC_debugging_started), 0); ++ op = 0; ++ for (i = 0; i < nblocks; i++) ++ op = GC_build_fl(hbs[i], lw, ++ (ok -> ok_init || GC_debugging_started), (ptr_t)op); + + *result = op; + GC_acquire_mark_lock(); +@@ -467,7 +484,10 @@ + return; + } + # endif +- op = GC_build_fl(h, lw, (ok -> ok_init || GC_debugging_started), 0); ++ op = 0; ++ for (i = 0; i < nblocks; i++) ++ op = GC_build_fl(hbs[i], lw, ++ (ok -> ok_init || GC_debugging_started), (ptr_t)op); + goto out; + } + } +diff -ur a/misc.c b/misc.c +--- a/misc.c 2026-07-03 12:56:51.390341791 +0200 ++++ b/misc.c 2026-07-09 20:00:13.788169293 +0200 +@@ -1157,6 +1157,14 @@ + } + } + { ++ char * many_blocks_string = GETENV("GC_MALLOC_MANY_BLOCKS"); ++ if (many_blocks_string != NULL) { ++ int many_blocks = atoi(many_blocks_string); ++ if (many_blocks > 0 && many_blocks <= GC_MANY_BLOCKS_MAX) ++ GC_many_blocks = many_blocks; ++ } ++ } ++ { + char * space_divisor_string = GETENV("GC_FREE_SPACE_DIVISOR"); + if (space_divisor_string != NULL) { + int space_divisor = atoi(space_divisor_string); +diff -ur a/os_dep.c b/os_dep.c +--- a/os_dep.c 2026-07-03 12:56:51.390341791 +0200 ++++ b/os_dep.c 2026-07-04 01:00:23.191858912 +0200 +@@ -2286,6 +2286,24 @@ + if (((word)result % HBLKSIZE) != 0) + ABORT( + "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE."); ++# if defined(LINUX) && defined(MADV_HUGEPAGE) ++ /* Optionally ask for transparent huge pages. The heap is large */ ++ /* and densely touched, so 2 MiB pages greatly reduce first-touch */ ++ /* page-fault and TLB overhead. However, under the "madvise" THP */ ++ /* policy, faults in a MADV_HUGEPAGE region perform synchronous */ ++ /* memory compaction, which can stall for a long time when */ ++ /* physical memory is fragmented. Hence this is opt-in via the */ ++ /* GC_MADVISE_HUGEPAGES environment variable; setting the system */ ++ /* THP policy to "always" (which takes huge pages opportunistically */ ++ /* without direct compaction) is generally preferable. */ ++ { ++ static int use_hugepages = -1; ++ if (use_hugepages < 0) ++ use_hugepages = GETENV("GC_MADVISE_HUGEPAGES") != NULL; ++ if (use_hugepages) ++ (void)madvise(result, bytes, MADV_HUGEPAGE); ++ } ++# endif + return((ptr_t)result); + } + # endif /* !MSWIN_XBOX1 */ diff --git a/packaging/patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch b/packaging/patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch new file mode 100644 index 000000000000..8cef53ce8dbb --- /dev/null +++ b/packaging/patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch @@ -0,0 +1,133 @@ +Don't abort pack building when worker thread creation fails. + +Previously, if `git_thread_create` failed in `ll_find_deltas` (e.g. with +EAGAIN due to transient resource exhaustion), it returned an error while +the already-created worker threads were still running. The caller +(`git_packbuilder_write_buf`) would then propagate the error and the +packbuilder would typically be freed, causing the orphaned workers to +crash on freed memory (observed as SIGSEGV in `git_odb_read` → +`git_cache_get_raw` → `pthread_rwlock_rdlock`). + +Instead, continue with however many threads were successfully created. +The work-stealing loop redistributes the work of partitions that have no +thread (`started == false`) to the running threads; such partitions are +stolen from regardless of size, and in their entirety, so every object +is still searched for deltas. If no threads could be created at all, +fall back to the single-threaded delta search. + +Also turn the per-thread mutex lock failure in the work-stealing loop +into a GIT_ASSERT, consistent with the neighboring progress lock/unlock +checks. In particular it must not free the `thread_params` array, since +running worker threads still reference it. + +Upstream: https://github.com/libgit2/libgit2 (present on main as of 2026-07-16) +--- a/src/libgit2/pack-objects.c ++++ b/src/libgit2/pack-objects.c +@@ -1137,6 +1137,9 @@ + /* A pb->progress_cb can stop the packing process by returning an error. + When that happens, all threads observe the error and stop voluntarily. */ + bool stopped; ++ ++ /* Whether the worker thread for this partition was created. */ ++ bool started; + }; + + static void *threaded_find_deltas(void *arg) +@@ -1186,6 +1189,8 @@ + size_t list_size, size_t window, size_t depth) + { + struct thread_params *p; ++ git_pobject **whole_list = list; ++ size_t whole_list_size = list_size; + size_t i; + int ret, active_threads = 0; + +@@ -1213,6 +1218,7 @@ + p[i].working = 1; + p[i].data_ready = 0; + p[i].stopped = 0; ++ p[i].started = 0; + + /* try to split chunks on "path" boundaries */ + while (sub_size && sub_size < list_size && +@@ -1239,13 +1245,37 @@ + ret = git_thread_create(&p[i].thread, + threaded_find_deltas, &p[i]); + if (ret) { +- git_error_set(GIT_ERROR_THREAD, "unable to create thread"); +- return -1; ++ /* ++ * Thread creation may fail due to transient resource ++ * exhaustion (e.g. EAGAIN). Proceed with the threads ++ * created so far: this partition keeps `working` set, ++ * so it is never picked as a work-stealing target, and ++ * the loop below hands all of its work to the running ++ * threads (`started` remains false). ++ * ++ * Returning an error here instead would leave the ++ * already-created threads running while the caller ++ * frees the packbuilder, resulting in a use-after-free. ++ */ ++ git_cond_free(&p[i].cond); ++ git_mutex_free(&p[i].mutex); ++ continue; + } ++ p[i].started = 1; + active_threads++; + } + + /* ++ * If no worker threads could be created, fall back to doing the ++ * delta search inline. No thread has consumed any work at this ++ * point, so the whole object list is still intact. ++ */ ++ if (!active_threads) { ++ git__free(p); ++ return find_deltas(pb, whole_list, &whole_list_size, window, depth); ++ } ++ ++ /* + * Now let's wait for work completion. Each time a thread is done + * with its work, we steal half of the remaining work from the + * thread with the largest number of unprocessed objects and give +@@ -1274,16 +1304,23 @@ + + /* At this point we hold the progress lock and have located + * a thread to receive more work. We still need to locate a +- * thread from which to steal work (the victim). */ ++ * thread from which to steal work (the victim). A partition ++ * whose thread could not be created has nothing consuming ++ * it, so it may be stolen from regardless of its size. */ + for (i = 0; i < pb->nr_threads; i++) +- if (p[i].remaining > 2*window && ++ if (p[i].remaining > (p[i].started ? 2*window : 0) && + (!victim || victim->remaining < p[i].remaining)) + victim = &p[i]; + + if (victim && !target->stopped) { +- sub_size = victim->remaining / 2; ++ /* Steal half of the victim's work, or all of it if ++ * the victim has no thread consuming it. In the ++ * latter case there is also no point in splitting on ++ * a "path" boundary, since the partition is not ++ * shared with another thread. */ ++ sub_size = victim->remaining / (victim->started ? 2 : 1); + list = victim->list + victim->list_size - sub_size; +- while (sub_size && list[0]->hash && ++ while (victim->started && sub_size && list[0]->hash && + list[0]->hash == list[-1]->hash) { + list++; + sub_size--; +@@ -1307,11 +1344,7 @@ + target->working = 1; /* even when target->stopped, so that we don't process this thread again */ + GIT_ASSERT(git_packbuilder__progress_unlock(pb) == 0); + +- if (git_mutex_lock(&target->mutex)) { +- git_error_set(GIT_ERROR_THREAD, "unable to lock packfile condition mutex"); +- git__free(p); +- return -1; +- } ++ GIT_ASSERT(git_mutex_lock(&target->mutex) == 0); + + target->data_ready = 1; + git_cond_signal(&target->cond); diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix new file mode 100644 index 000000000000..0d89b33714f2 --- /dev/null +++ b/packaging/release-jobs.nix @@ -0,0 +1,81 @@ +# Hydra jobset containing only the artifacts consumed by +# `maintainers/upload-release.pl`, so a release can be cut without +# waiting on the full `hydraJobs` CI matrix. +# +# Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake +# jobsets to `outputs.hydraJobs`; we re-enter the flake via +# `builtins.getFlake` so derivations stay identical to the flake jobset +# and share builds through the binary cache. +# +# Hydra jobset configuration: +# Identifier: maintenance--release +# Type: Legacy +# Nix expression: packaging/release-jobs.nix in input `src` +# Inputs: +# src (Git checkout) https://github.com/NixOS/nix +{ + src ? { + outPath = ./..; + }, +}: +let + # Fetch by GitHub ref rather than the bare store path Hydra hands us, + # so `rev`/`lastModified` (and thus the version suffix) match the flake + # jobset and derivations are shared. + flake = builtins.getFlake ( + if src ? rev then + "github:NixOS/nix/${src.rev}" + else + # Local evaluation / testing. + builtins.unsafeDiscardStringContext (toString src) + ); + inherit (flake) hydraJobs; + inherit (flake.inputs.nixpkgs) lib; + + jobs = { + # `nix-everything` per system: provides the store paths for + # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via + # its `doc` output. + build.nix-everything = hydraJobs.build.nix-everything; + buildCross.nix-everything = { + # Only the cross targets that end up in `fallback-paths.nix`. + inherit (hydraJobs.buildCross.nix-everything) + riscv64-unknown-linux-gnu + ; + }; + + inherit (hydraJobs) + manual + binaryTarball + binaryTarballCross + installerScript + installerScriptForGHA + dockerImage + ; + + # Aggregate gating job: green ⇒ every artifact the upload script + # needs is available. `upload-release` can wait on this single job + # instead of the whole evaluation. Constituents are referenced by + # job *name* so that an evaluation failure in one of them does not + # take down the aggregate's own evaluation. + release = flake.inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate { + name = "nix-release-${flake.packages.x86_64-linux.nix-everything.version}"; + meta.description = "Artifacts required for a Nix release"; + constituents = + let + collectJobNames = + prefix: x: + if lib.isDerivation x then + [ prefix ] + else if lib.isAttrs x then + lib.concatLists ( + lib.mapAttrsToList (n: collectJobNames (if prefix == "" then n else "${prefix}.${n}")) x + ) + else + [ ]; + in + collectJobNames "" (builtins.removeAttrs jobs [ "release" ]); + }; + }; +in +jobs diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix new file mode 100644 index 000000000000..98aec45e9e53 --- /dev/null +++ b/packaging/rust-installer/default.nix @@ -0,0 +1,88 @@ +# `NixOS/nix-installer` built with *this* Nix closure embedded, so +# Hydra/CI can dogfood the Rust installer without the (removed) +# `--nix-package-url` knob. +{ + lib, + stdenv, + buildPackages, + runCommand, + rustPlatform, + fetchFromGitHub, + tarball, +}: + +let + installerVersion = "2.34.6"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "nix-installer"; + tag = installerVersion; + hash = "sha256-aTaz8EtHexvke7tGr5MfeKy9g7AraIAFN+dPApm+fds="; + }; + + # Bare binary: no Nix closure yet. Appended below via `pack`, so the + # (expensive) Rust compile is independent of the embedded Nix and + # stays cacheable across Nix revisions. + bare = rustPlatform.buildRustPackage { + pname = "nix-installer-bare"; + version = installerVersion; + + inherit src; + + cargoHash = "sha256-/mNXkeZVuYsqd0TiUa7bzSP4xpKh0Fqga9EpasPbrzU="; + + doCheck = false; + + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + ''; + }; +in + +runCommand "nix-installer-${tarball.passthru.nixVersion}" + { + nativeBuildInputs = [ + buildPackages.python3 + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + buildPackages.darwin.sigtool + buildPackages.darwin.cctools + ]; + + # The appended payload contains store-path strings on purpose; don't + # let the reference scanner pull the whole Nix closure into this + # derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + + passthru = { inherit bare; }; + + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; + } + '' + mkdir -p $out/bin $out/nix-support + + python3 ${src}/scripts/pack \ + --input ${bare}/bin/nix-installer \ + --tarball ${tarball}/nix.tar.zst \ + --nix-store-path ${tarball.passthru.nixStorePath} \ + --cacert-store-path ${tarball.passthru.cacertStorePath} \ + --nix-version ${tarball.passthru.nixVersion} \ + --output $out/bin/nix-installer + + install -m755 ${bare}/bin/nix-installer.sh $out/bin/nix-installer.sh + + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + '' diff --git a/packaging/rust-installer/tarball.nix b/packaging/rust-installer/tarball.nix new file mode 100644 index 000000000000..4f236e0215bb --- /dev/null +++ b/packaging/rust-installer/tarball.nix @@ -0,0 +1,50 @@ +# Zstd-compressed Nix closure in the layout expected by +# `NixOS/nix-installer` (`include_bytes!` at build time). +{ + lib, + stdenv, + runCommand, + buildPackages, + zstd, + nix, + cacert, +}: + +let + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ + nix + cacert + ]; + }; +in + +runCommand "nix-installer-tarball-${nix.version}" + { + nativeBuildInputs = [ zstd ]; + + passthru = { + nixStorePath = nix.outPath; + cacertStorePath = cacert.outPath; + nixVersion = nix.version; + }; + } + '' + mkdir -p $out + + dir=nix-${nix.version}-${stdenv.hostPlatform.system} + + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + + tar cf - \ + --sort=name \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) \ + | zstd -19 -T1 -o $out/nix.tar.zst + '' diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock new file mode 100644 index 000000000000..651372b84652 --- /dev/null +++ b/packaging/secure-packages/flake.lock @@ -0,0 +1,132 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "revCount": 480, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.480%2Brev-17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e/019f2195-dee5-7233-9747-eca0c27f7406/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "revCount": 1231, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1231%2Brev-43b3c1ab9d40fb1dbb008f451988a91e375825e9/019f7135-8fdf-76f0-b1a1-d2c67e91af8d/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941" + } + }, + "nix": { + "inputs": { + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", + "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "path": "../..", + "type": "path" + }, + "original": { + "path": "../..", + "type": "path" + }, + "parent": [] + }, + "nixpkgs": { + "locked": { + "lastModified": 1785435372, + "narHash": "sha256-veAo2hsRG1Off47F+1HHgwarJVo/FjXUFv3A40ZUViY=", + "rev": "e87516da91e53b694a46eda0de22b782eb008d82", + "revCount": 1006668, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006668%2Brev-e87516da91e53b694a46eda0de22b782eb008d82/019fb45e-7a22-75f0-a9f9-a895378372a0/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/DeterminateSystems/secure-packages-26.05/0" + } + }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "root": { + "inputs": { + "nix": "nix" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/packaging/secure-packages/flake.nix b/packaging/secure-packages/flake.nix new file mode 100644 index 000000000000..723eb14fe414 --- /dev/null +++ b/packaging/secure-packages/flake.nix @@ -0,0 +1,6 @@ +{ + inputs.nix.url = "../.."; + inputs.nix.inputs.nixpkgs.url = "https://flakehub.com/f/DeterminateSystems/secure-packages-26.05/0"; + + outputs = { self, nix }: nix.outputs; +} diff --git a/packaging/sentry-native.nix b/packaging/sentry-native.nix new file mode 100644 index 000000000000..0e21e9be55c2 --- /dev/null +++ b/packaging/sentry-native.nix @@ -0,0 +1,54 @@ +{ + lib, + stdenv, + fetchgit, + cmake, + curl, + pkg-config, + python3, + darwin, +}: + +stdenv.mkDerivation rec { + pname = "sentry-native"; + version = "0.13.5"; + + src = fetchgit { + url = "https://github.com/getsentry/sentry-native"; + tag = version; + hash = "sha256-vDBI6lB1DMLleAgRCfsHvTSdtmXOzvJSaNAt+NwOd3c="; + fetchSubmodules = true; + }; + + dontFixCmake = true; + + nativeBuildInputs = [ + cmake + pkg-config + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + python3 + darwin.bootstrap_cmds + ]; + + postPatch = '' + # Borrowed from psutil: stick to the old SDK name for now. + substituteInPlace external/crashpad/util/mac/mac_util.cc \ + --replace-fail kIOMainPortDefault kIOMasterPortDefault + ''; + + buildInputs = [ + curl + ]; + + cmakeBuildType = "RelWithDebInfo"; + + cmakeFlags = [ ]; + + outputs = [ + "out" + "dev" + ]; + + separateDebugInfo = true; +} diff --git a/packaging/wasmtime.nix b/packaging/wasmtime.nix new file mode 100644 index 000000000000..d2c2b95f6072 --- /dev/null +++ b/packaging/wasmtime.nix @@ -0,0 +1,74 @@ +# Stripped-down version of https://github.com/NixOS/nixpkgs/blob/master/pkgs/by-name/wa/wasmtime/package.nix, +# license: https://github.com/NixOS/nixpkgs/blob/master/COPYING +{ + lib, + stdenv, + rust, + fetchFromGitHub, + cmake, + enableShared ? !stdenv.hostPlatform.isStatic, + enableStatic ? stdenv.hostPlatform.isStatic, +}: +rust.packages.stable.rustPlatform.buildRustPackage (finalAttrs: { + pname = "wasmtime"; + version = "40.0.2"; + + src = fetchFromGitHub { + owner = "bytecodealliance"; + repo = "wasmtime"; + tag = "v${finalAttrs.version}"; + hash = "sha256-4y9WpCdyuF/Tp2k/1d5rZxwYunWNdeibEsFgHcBC52Q="; + fetchSubmodules = true; + }; + + # Disable cargo-auditable until https://github.com/rust-secure-code/cargo-auditable/issues/124 is solved. + auditable = false; + + cargoHash = "sha256-aTPgnuBvOIqg1+Sa2ZLdMTLujm8dKGK5xpZ3qHpr3f8="; + cargoBuildFlags = [ + "--package" + "wasmtime-c-api" + "--no-default-features" + "--features cranelift,wasi,pooling-allocator,wat,demangle,gc-null" + ]; + + outputs = [ + "out" + "lib" + ]; + + nativeBuildInputs = [ + cmake + ]; + + doCheck = + with stdenv.buildPlatform; + # SIMD tests are only executed on platforms that support all + # required processor features (e.g. SSE3, SSSE3 and SSE4.1 on x86_64): + # https://github.com/bytecodealliance/wasmtime/blob/v9.0.0/cranelift/codegen/src/isa/x64/mod.rs#L220 + (isx86_64 -> sse3Support && ssse3Support && sse4_1Support) + && + # The dependency `wasi-preview1-component-adapter` fails to build because of: + # error: linker `rust-lld` not found + !isAarch64; + + postInstall = + let + inherit (stdenv.hostPlatform.rust) cargoShortTarget; + in + '' + moveToOutput lib $lib + ${lib.optionalString (!enableShared) "rm -f $lib/lib/*.so{,.*}"} + ${lib.optionalString (!enableStatic) "rm -f $lib/lib/*.a"} + + # copy the build.rs generated c-api headers + # https://github.com/rust-lang/cargo/issues/9661 + mkdir -p $out + cp -r target/${cargoShortTarget}/release/build/wasmtime-c-api-impl-*/out/include $out/include + '' + + lib.optionalString stdenv.hostPlatform.isDarwin '' + install_name_tool -id \ + $lib/lib/libwasmtime.dylib \ + $lib/lib/libwasmtime.dylib + ''; +}) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f33..538a5e74d5ee 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea6c..ae6625e1bc41 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index f17e4c2af3b9..73f389b6ff16 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -28,14 +28,15 @@ fi OS="$(uname -s)" -# macOS support for 10.12.6 or higher +# Since nixpkgs 25.11 the minimum deployment target is macOS 14.0 if [ "$OS" = "Darwin" ]; then + # shellcheck disable=SC2034 IFS='.' read -r macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF - if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + if [ "$macos_major" -lt 14 ]; then # patch may not be present; command substitution for simplicity - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 14.0 or higher" exit 1 fi fi diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd2459..93cb3c45a55b 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6fc..201a56438950 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 918f4bbd9e9e..000000000000 --- a/shell.nix +++ /dev/null @@ -1,3 +0,0 @@ -(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { - src = ./.; -}).shellNix diff --git a/src/perl/.version b/src/clang-tidy-plugin/.version similarity index 100% rename from src/perl/.version rename to src/clang-tidy-plugin/.version diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build new file mode 100644 index 000000000000..7896bd085479 --- /dev/null +++ b/src/clang-tidy-plugin/meson.build @@ -0,0 +1,50 @@ +project( + 'nix-clang-tidy-plugin', + 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++23', + 'warning_level=2', + ], + meson_version : '>= 1.8', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/deps-lists') +subdir('nix-meson-build-support/common') + +# clang-tidy plugins require LLVM (clang headers provided via buildInputs) +llvm_dep = dependency('LLVM', version : '>= 16', required : true) + +sources = files( + 'nix-clang-tidy-checks.cc', +) + +# Build as a shared module (plugin) that can be loaded by clang-tidy --load +nix_clang_tidy_plugin = shared_module( + 'nix-clang-tidy', + sources, + dependencies : [ llvm_dep ], + install : true, +) + +# Export plugin path for consumers (nix-meson-build-support/common/clang-tidy/meson.build). +# In monolithic builds: override_dependency provides the build-dir path. +# In per-component Nix builds: the .pc file provides the installed path. +meson.override_dependency( + 'nix-clang-tidy', + declare_dependency( + variables : {'plugin_path' : nix_clang_tidy_plugin.full_path()}, + ), +) +import('pkgconfig').generate( + nix_clang_tidy_plugin, + filebase : 'nix-clang-tidy', + name : 'nix-clang-tidy', + description : 'Custom clang-tidy checks for Nix', + variables : { + 'plugin_path' : get_option('prefix') / get_option('libdir') / 'libnix-clang-tidy.so', + }, +) diff --git a/src/clang-tidy-plugin/nix-clang-tidy-checks.cc b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc new file mode 100644 index 000000000000..64ec63c2d319 --- /dev/null +++ b/src/clang-tidy-plugin/nix-clang-tidy-checks.cc @@ -0,0 +1,34 @@ +/** + * @brief Custom clang-tidy checks for the Nix project. + * + * This module registers custom clang-tidy checks specific to the Nix codebase. + * To add a new check: + * 1. Create check-name.hh and check-name.cc in this directory + * 2. Include the header here + * 3. Register the check in addCheckFactories() + * 4. Add the source file to meson.build + * 5. Enable the check in .clang-tidy (e.g., nix-checkname) + */ + +#include +#include + +namespace nix::clang_tidy { + +using namespace clang; +using namespace clang::tidy; + +class NixClangTidyChecks : public ClangTidyModule +{ +public: + void addCheckFactories([[maybe_unused]] ClangTidyCheckFactories & CheckFactories) override + { + // Custom checks will be registered here. + // Example: + // CheckFactories.registerCheck("nix-my-custom-check"); + } +}; + +static ClangTidyModuleRegistry::Add X("nix-module", "Adds Nix-specific checks"); + +} // namespace nix::clang_tidy diff --git a/src/clang-tidy-plugin/nix-meson-build-support b/src/clang-tidy-plugin/nix-meson-build-support new file mode 120000 index 000000000000..0b140f56bdee --- /dev/null +++ b/src/clang-tidy-plugin/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/clang-tidy-plugin/package.nix b/src/clang-tidy-plugin/package.nix new file mode 100644 index 000000000000..d55a4b0b4ec0 --- /dev/null +++ b/src/clang-tidy-plugin/package.nix @@ -0,0 +1,46 @@ +{ + lib, + mkMesonDerivation, + + pkg-config, + llvmPackages, + + # Configuration Options + + version, +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-clang-tidy-plugin"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + pkg-config + llvmPackages.llvm.dev + ]; + + buildInputs = [ + llvmPackages.libclang + llvmPackages.clang-unwrapped.dev # for ClangTidyModule.h + ]; + + meta = { + description = "Custom clang-tidy checks for the Nix codebase"; + platforms = lib.platforms.unix; + }; +}) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813fa..ea8c069e9545 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -42,6 +42,8 @@ INPUT = \ @src@/src/libexpr-c \ @src@/src/libflake-c \ @src@/src/libstore-c \ + @src@/src/libfetchers-c \ + @src@/src/libmain-c \ @src@/src/external-api-docs/README.md FILE_PATTERNS = nix_api_*.h *.md @@ -61,3 +63,4 @@ USE_MDFILE_AS_MAINPAGE = @src@/src/external-api-docs/README.md WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index cba6f646b93b..d96da3863e6f 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -14,6 +14,7 @@ doxygen_cfg = configure_file( 'PROJECT_NUMBER' : meson.project_version(), 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'src' : fs.parent(fs.parent(meson.project_source_root())), + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index b194e16d4608..79fe692fa572 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-external-api-docs"; + pname = "determinate-nix-external-api-docs"; inherit version; workDir = ./.; @@ -31,7 +31,9 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libfetchers-c) (cpp ../libflake-c) + (cpp ../libmain-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in index 2769edd9f727..aa81d77c5684 100644 --- a/src/internal-api-docs/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -112,3 +112,4 @@ PREDEFINED = DOXYGEN_SKIP WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index daab4c93c309..3976e4c3f81a 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -15,6 +15,7 @@ doxygen_cfg = configure_file( 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'BUILD_ROOT' : meson.build_root(), 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index 6c4f354aee5c..636c19653eab 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-internal-api-docs"; + pname = "determinate-nix-internal-api-docs"; inherit version; workDir = ./.; diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a1b525bc51e4..66dc9b758a44 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -51,6 +51,13 @@ schemas = [ 'simple.json', ], }, + { + 'stem' : 'signature', + 'schema' : schema_dir / 'signature-v2.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', @@ -62,12 +69,9 @@ schemas = [ }, { 'stem' : 'build-trace-entry', - 'schema' : schema_dir / 'build-trace-entry-v2.yaml', + 'schema' : schema_dir / 'build-trace-entry-v3.yaml', 'files' : [ 'simple.json', - # The field is no longer supported, but we want to show that we - # ignore it during parsing. - 'with-dependent-realisations.json', 'with-signature.json', ], }, @@ -94,6 +98,17 @@ schemas += [ 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', + 'try-resolve' / 'no-inputs-before.json', + 'try-resolve' / 'with-inputs-before.json', + 'try-resolve' / 'resolution-failure-before.json', + ], + }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-resolved-v4.yaml', + 'files' : [ + 'try-resolve' / 'no-inputs-after.json', + 'try-resolve' / 'with-inputs-after.json', ], }, { @@ -154,20 +169,20 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', - 'json-2' / 'empty_pure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_pure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', ], }, { @@ -182,32 +197,32 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'empty_pure.json', + 'json-3' / 'pure.json', + 'json-3' / 'empty_pure.json', ], }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/impure', 'files' : [ - 'json-2' / 'impure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', + 'json-3' / 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/narInfo', 'files' : [ - 'json-2' / 'impure.json', + 'json-3' / 'impure.json', ], }, ] diff --git a/src/json-schema-checks/signature/simple.json b/src/json-schema-checks/signature/simple.json new file mode 100644 index 000000000000..abf69dfa0f92 --- /dev/null +++ b/src/json-schema-checks/signature/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f1849384c..dd8d8a0f7e36 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,12 +1,12 @@ #include "nix/cmd/built-path.hh" +#include "nix/store/build-result.hh" #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" +#include "nix/store/outputs-query.hh" #include "nix/util/comparator.hh" #include -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality @@ -108,23 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; - auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); - } else { - res.insert(outputPath); - } + /* Use a custom callback to collect realisations as they're queried. */ + deepQueryPartialDerivationOutput( + store, p.drvPath->outPath(), outputName, nullptr, [&](const DrvOutput & drvOutput) { + auto realisation = store.queryRealisation(drvOutput); + if (realisation) + res.insert(Realisation{*realisation, drvOutput}); + return realisation; + }); + res.insert(outputPath); } }, }, @@ -132,4 +125,50 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const return res; } +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd{ + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built{ + .drvPath = make_ref(std::move(drvPath)), + .output = {bfd.output, outputPath}, + }; + }, + }, + b.raw()); +} + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store) +{ + auto success = result.tryGetSuccess(); + assert(success); + return std::visit( + overloaded{ + [&](const DerivedPath::Built & bfd) { + std::map outputs; + for (auto & [outputName, realisation] : success->builtOutputs) + outputs.emplace(outputName, realisation.outPath); + BuiltPath bp = BuiltPath::Built{ + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }; + return bp; + }, + [&](const DerivedPath::Opaque & bo) { + BuiltPath bp = BuiltPath::Opaque{bo.path}; + return bp; + }, + }, + result.path.raw()); +} + } // namespace nix diff --git a/src/libcmd/builtin-flake-schemas.nix b/src/libcmd/builtin-flake-schemas.nix new file mode 100644 index 000000000000..8d6cbccbd5f3 --- /dev/null +++ b/src/libcmd/builtin-flake-schemas.nix @@ -0,0 +1,495 @@ +{ + description = "Schemas for well-known Nix flake output types"; + + outputs = + { self }: + let + mapAttrsToList = f: attrs: map (name: f name attrs.${name}) (builtins.attrNames attrs); + + checkModule = + module_: + let + module = if builtins.isPath module_ then import module_ else module_; + in + builtins.isAttrs module || builtins.isFunction module; + + mkApp = system: app: { + forSystems = [ system ]; + evalChecks.isValidApp = + app ? type + && app.type == "app" + && app ? program + && builtins.isString app.program + && + builtins.removeAttrs app [ + "type" + "program" + "meta" + ] == { }; + what = "app"; + shortDescription = app.meta.description or ""; + }; + + mkPackage = isFlakeCheck: what: system: package: { + forSystems = [ system ]; + shortDescription = package.meta.description or ""; + derivationAttrPath = [ ]; + inherit what isFlakeCheck; + }; + + singleDerivationInventory = + what: isFlakeCheck: output: + self.lib.mkChildren (builtins.mapAttrs (mkPackage isFlakeCheck what) output); + + schemasSchema = doc: { + version = 1; + inherit doc; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (schemaName: schemaDef: { + shortDescription = "A schema checker for the `${schemaName}` flake output"; + evalChecks.isValidSchema = + schemaDef.version or 0 == 1 + && schemaDef ? doc + && builtins.isString (schemaDef.doc) + && schemaDef ? inventory + && builtins.isFunction (schemaDef.inventory); + what = "flake schema"; + }) output + ); + }; + + appsSchema = { + version = 1; + doc = '' + The `apps` output provides commands available via `nix run`. + ''; + roles.nix-run = { }; + appendSystem = true; + defaultAttrPath = [ "default" ]; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (system: apps: { + forSystems = [ system ]; + children = builtins.mapAttrs (appName: app: mkApp system app) apps; + }) output + ); + }; + + defaultAppSchema = { + version = 1; + doc = '' + **DEPRECATED**. Use `apps..default` instead. + ''; + roles.nix-run = { }; + appendSystem = true; + defaultAttrPath = [ ]; + inventory = output: self.lib.mkChildren (builtins.mapAttrs mkApp output); + }; + + packagesSchema = { + version = 1; + doc = '' + The `packages` flake output contains packages that can be added to a shell using `nix shell`. + ''; + roles.nix-build = { }; + roles.nix-run = { }; + roles.nix-develop = { }; + roles.nix-search = { }; + appendSystem = true; + defaultAttrPath = [ "default" ]; + inventory = self.lib.derivationsInventory "package" false; + }; + + defaultPackageSchema = { + version = 1; + doc = '' + **DEPRECATED**. Use `packages..default` instead. + ''; + roles.nix-build = { }; + roles.nix-run = { }; + roles.nix-develop = { }; + roles.nix-search = { }; + appendSystem = true; + defaultAttrPath = [ ]; + inventory = singleDerivationInventory "package" false; + }; + + ociImagesSchema = { + version = 1; + doc = '' + The `ociImages` flake output contains derivations that build valid Open Container Initiative images. + ''; + inventory = self.lib.derivationsInventory "OCI image" false; + }; + + legacyPackagesSchema = { + version = 1; + doc = '' + The `legacyPackages` flake output is similar to `packages` but different in that it can be nested and thus contain attribute sets that contain more packages. + Since enumerating packages in nested attribute sets can be inefficient, you should favor `packages` over `legacyPackages`. + ''; + roles.nix-build = { }; + roles.nix-run = { }; + roles.nix-search = { }; + roles.nix-develop = { }; + appendSystem = true; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (systemType: packagesForSystem: { + forSystems = [ systemType ]; + isLegacy = true; + children = + let + recurse = + prefix: attrs: + builtins.mapAttrs ( + attrName: attrs: + # Necessary to deal with `AAAAAASomeThingsFailToEvaluate` etc. in Nixpkgs. + self.lib.try ( + if attrs.type or null == "derivation" then + { + forSystems = [ attrs.system ]; + shortDescription = attrs.meta.description or ""; + derivationAttrPath = [ ]; + what = "package"; + } + else + # Recurse at the first and second levels, or if the + # recurseForDerivations attribute if set. + if attrs.recurseForDerivations or false then + { + children = recurse (prefix + attrName + ".") attrs; + } + else + { + what = "unknown"; + } + ) (throw "failed") + ) attrs; + in + # The top-level cannot be a derivation. + assert packagesForSystem.type or null != "derivation"; + recurse (systemType + ".") packagesForSystem; + }) output + ); + }; + + checksSchema = { + version = 1; + doc = '' + The `checks` flake output contains derivations that will be built by `nix flake check`. + ''; + # FIXME: add role + inventory = self.lib.derivationsInventory "CI test" true; + }; + + devShellsSchema = { + version = 1; + doc = '' + The `devShells` flake output contains derivations that provide a development environment for `nix develop`. + ''; + roles.nix-develop = { }; + appendSystem = true; + defaultAttrPath = [ "default" ]; + inventory = self.lib.derivationsInventory "development environment" false; + }; + + devShellSchema = { + version = 1; + doc = '' + **DEPRECATED**. Use `devShells..default` instead. + ''; + roles.nix-develop = { }; + appendSystem = true; + defaultAttrPath = [ ]; + inventory = singleDerivationInventory "development environment" false; + }; + + formatterSchema = { + version = 1; + doc = '' + The `formatter` output specifies the package to use to format the project. + ''; + roles.nix-fmt = { }; + appendSystem = true; + defaultAttrPath = [ ]; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (system: formatter: { + forSystems = [ system ]; + shortDescription = formatter.meta.description or ""; + derivationAttrPath = [ ]; + what = "formatter"; + isFlakeCheck = false; + }) output + ); + }; + + templatesSchema = { + version = 1; + doc = '' + The `templates` output provides project templates. + ''; + roles.nix-template = { }; + defaultAttrPath = [ "default" ]; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (templateName: template: { + shortDescription = template.description or ""; + evalChecks.isValidTemplate = + template ? path + && builtins.isPath template.path + && template ? description + && builtins.isString template.description; + what = "template"; + }) output + ); + }; + + hydraJobsSchema = { + version = 1; + doc = '' + The `hydraJobs` flake output defines derivations to be built by the Hydra continuous integration system. + ''; + allowIFD = false; + inventory = + output: + let + recurse = + prefix: attrs: + self.lib.mkChildren ( + builtins.mapAttrs ( + attrName: attrs: + if attrs.type or null == "derivation" then + { + forSystems = [ attrs.system ]; + shortDescription = attrs.meta.description or ""; + derivationAttrPath = [ ]; + what = "Hydra CI test"; + } + else + recurse (prefix + attrName + ".") attrs + ) attrs + ); + in + # The top-level cannot be a derivation. + assert output.type or null != "derivation"; + recurse "" output; + }; + + overlaysSchema = { + version = 1; + doc = '' + The `overlays` flake output defines ["overlays"](https://nixos.org/manual/nixpkgs/stable/#chap-overlays) that can be plugged into Nixpkgs. + Overlays add additional packages or modify or replace existing packages. + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (overlayName: overlay: { + what = "Nixpkgs overlay"; + evalChecks.isOverlay = + # FIXME: should try to apply the overlay to an actual + # Nixpkgs. But we don't have access to a nixpkgs + # flake here. Maybe this schema should be moved to the + # nixpkgs flake, where it does have access. + if !builtins.isFunction overlay then + throw "Overlay is not a function. It should be structured like: `final: previous: { /* ... */ }`." + else + true; + }) output + ); + }; + + nixosConfigurationsSchema = { + version = 1; + doc = '' + The `nixosConfigurations` flake output defines [NixOS system configurations](https://nixos.org/manual/nixos/stable/#ch-configuration). + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (configName: machine: { + what = "NixOS configuration"; + derivationAttrPath = [ + "config" + "system" + "build" + "toplevel" + ]; + forSystems = [ machine.pkgs.stdenv.system ]; + }) output + ); + }; + + nixosModulesSchema = { + version = 1; + doc = '' + The `nixosModules` flake output defines importable [NixOS modules](https://nixos.org/manual/nixos/stable/#sec-writing-modules). + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (moduleName: module: { + what = "NixOS module"; + evalChecks.isFunctionOrAttrs = checkModule module; + }) output + ); + }; + + homeConfigurationsSchema = { + version = 1; + doc = '' + The `homeConfigurations` flake output defines [Home Manager configurations](https://github.com/nix-community/home-manager). + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (configName: this: { + what = "Home Manager configuration"; + derivationAttrPath = [ "activationPackage" ]; + forSystems = [ this.activationPackage.system ]; + }) output + ); + }; + + homeModulesSchema = { + version = 1; + doc = '' + The `homeModules` flake output defines importable [Home Manager](https://github.com/nix-community/home-manager) modules. + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (moduleName: module: { + what = "Home Manager module"; + evalChecks.isFunctionOrAttrs = checkModule module; + }) output + ); + }; + + darwinConfigurationsSchema = { + version = 1; + doc = '' + The `darwinConfigurations` flake output defines [nix-darwin configurations](https://github.com/nix-darwin/nix-darwin). + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (configName: this: { + what = "nix-darwin configuration"; + derivationAttrPath = [ "system" ]; + forSystems = [ this.system.system ]; + }) output + ); + }; + + darwinModulesSchema = { + version = 1; + doc = '' + The `darwinModules` flake output defines importable [nix-darwin modules](https://github.com/nix-darwin/nix-darwin). + ''; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs (moduleName: module: { + what = "nix-darwin module"; + evalChecks.isFunctionOrAttrs = checkModule module; + }) output + ); + }; + + bundlersSchema = { + version = 1; + doc = '' + The `bundlers` flake output defines ["bundlers"](https://nix.dev/manual/nix/latest/command-ref/new-cli/nix3-bundle) that transform derivation outputs into other formats, typically self-extracting executables or container images. + ''; + roles.nix-bundler = { }; + appendSystem = true; + defaultAttrPath = [ "default" ]; + inventory = + output: + self.lib.mkChildren ( + builtins.mapAttrs ( + system: bundlers: + let + forSystems = [ system ]; + in + { + inherit forSystems; + children = builtins.mapAttrs (bundlerName: bundler: { + inherit forSystems; + evalChecks.isValidBundler = builtins.isFunction bundler; + what = "bundler"; + }) bundlers; + } + ) output + ); + }; + + in + + { + # Helper functions + lib = { + try = + e: default: + let + res = builtins.tryEval e; + in + if res.success then res.value else default; + + mkChildren = children: { inherit children; }; + + derivationsInventory = + what: isFlakeCheck: output: + self.lib.mkChildren ( + builtins.mapAttrs (systemType: packagesForSystem: { + forSystems = [ systemType ]; + children = builtins.mapAttrs ( + packageName: mkPackage isFlakeCheck what systemType + ) packagesForSystem; + }) output + ); + }; + + exportedSchemas = { + schemas = schemasSchema '' + The `schemas` flake output is used to define and document flake outputs. + For the expected format, consult the Nix manual. + ''; + exportedSchemas = schemasSchema '' + The `exportedSchemas` flake output is used to define flake schemas that you + intend for other flakes to use. + ''; + apps = appsSchema; + defaultApp = defaultAppSchema; + packages = packagesSchema; + defaultPackage = defaultPackageSchema; + legacyPackages = legacyPackagesSchema; + checks = checksSchema; + devShells = devShellsSchema; + devShell = devShellSchema; + formatter = formatterSchema; + templates = templatesSchema; + hydraJobs = hydraJobsSchema; + overlays = overlaysSchema; + nixosConfigurations = nixosConfigurationsSchema; + nixosModules = nixosModulesSchema; + homeConfigurations = homeConfigurationsSchema; + homeModules = homeModulesSchema; + darwinConfigurations = darwinConfigurationsSchema; + darwinModules = darwinModulesSchema; + ociImages = ociImagesSchema; + bundlers = bundlersSchema; + }; + + schemas = self.exportedSchemas; + }; +} diff --git a/src/libcmd/call-flake-schemas.nix b/src/libcmd/call-flake-schemas.nix new file mode 100644 index 000000000000..1060ad56c292 --- /dev/null +++ b/src/libcmd/call-flake-schemas.nix @@ -0,0 +1,38 @@ +# The flake providing default schemas. +defaultSchemasFlake: + +# The flake whose contents we want to extract. +flake: + +let + + # Helper functions. + + mapAttrsToList = f: attrs: map (name: f name attrs.${name}) (builtins.attrNames attrs); + + outputNames = builtins.attrNames flake.outputs; + + schemas = flake.outputs.schemas or defaultSchemasFlake.exportedSchemas; + +in + +{ + outputs = flake.outputs; + + inventory = builtins.mapAttrs ( + outputName: _: + if schemas ? ${outputName} && schemas.${outputName}.version == 1 then + schemas.${outputName} + // ( + if flake.outputs ? ${outputName} then + { + output = schemas.${outputName}.inventory flake.outputs.${outputName}; + } + else + { + } + ) + else + { unknown = true; } + ) (schemas // flake.outputs); +} diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d57b76411328..045665ebf2fa 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -166,6 +166,13 @@ ref EvalCommand::getEvalStore() ref EvalCommand::getEvalState() { if (!evalState) { + if (startReplOnEvalErrors && evalSettings.evalCores != 1U) { + // Disable parallel eval if the debugger is enabled, since + // they're incompatible at the moment. + warn("using the debugger disables multi-threaded evaluation"); + evalSettings.evalCores = 1; + } + evalState = std::allocate_shared( traceable_allocator(), lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()); @@ -440,9 +447,34 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu void MixOutLinkBase::createOutLinksMaybe(const std::vector & buildables, ref & store) { - if (outLink != "") + createOutLinksMaybe(toBuiltPaths(buildables), store); +} + +void MixOutLinkBase::createOutLinksMaybe(const BuiltPaths & paths, ref & store) +{ + if (outLink) if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, toBuiltPaths(buildables), *store2); + createOutLinks(*outLink, paths, *store2); +} + +void MixPrintOutPaths::printOutPathsMaybe(const BuiltPaths & paths, ref store) +{ + if (!printOutputPaths) + return; + + logger->stop(); + for (auto & path : paths) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { logger->cout(store->printStorePath(bo.path)); }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + logger->cout(store->printStorePath(output.second)); + } + }, + }, + path); + } } } // namespace nix diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7e4bd7a162a3..a396a172816e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,9 +1,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval-settings.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/main/shared.hh" #include "nix/util/config-global.hh" -#include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" @@ -19,17 +17,12 @@ namespace nix { -fetchers::Settings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - EvalSettings evalSettings{ settings.readOnlyMode, { { "flake", [](EvalState & state, std::string_view rest) { - experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false); debug("fetching flake search path element '%s''", rest); @@ -186,7 +179,6 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::files } else if (hasPrefix(s, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store); diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 95fdf95ad00c..83aba78d8f4b 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,22 +1,56 @@ #include "nix/cmd/editor-for.hh" #include "nix/util/environment-variables.hh" #include "nix/util/source-path.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { -Strings editorFor(const SourcePath & file, uint32_t line) +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly) { auto path = file.getPhysicalPath(); - if (!path) - throw Error("cannot open '%s' in an editor because it has no physical path", file); - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); + OsString editor = getEnvOsNonEmpty(OS_STR("EDITOR")).value_or(OS_STR("cat")); + auto args = tokenizeString(editor); if (line > 0 - && (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos - || editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(path->string()); - return args; + && (editor.contains(OS_STR("emacs")) || editor.contains(OS_STR("nano")) || editor.contains(OS_STR("vim")) + || editor.contains(OS_STR("kak")))) + args.push_back(string_to_os_string(fmt("+%d", line))); + + if (path) { + args.push_back(path->native()); + return {std::move(args), AutoCloseFD{}, AutoDelete{}}; + } + + /* Resolve symlinks when creating a temporary. That's how a regular editor + would behave. Also GitSourceAccessor behaves poorly with symlinks in the + path and fails with "«...» does not exist". */ + auto file2 = file.resolveSymlinks(); + auto stat = file2.lstat(); + /* TODO: Maybe we should print a directory listing and open that instead? */ + if (stat.type != SourceAccessor::tRegular) + throw Error("can't open a file %s of type '%s'", file2.to_string(), stat.typeString()); + + auto tempDir = createTempDir(defaultTempDir(), "nix-edit", 0700); + AutoDelete autoDel(tempDir, /*recursive=*/true); + auto tempPath = tempDir / file2.path.baseName().value_or("nix-edit"); + + /* Create the file with the same name, so editors that recognise file + extensions spin up syntax highlighting and LSPs. */ + auto tempFd = openNewFileForWrite( + tempPath, + readOnly ? 0400 : 0600, + {.truncateExisting = false, .followSymlinksOnTruncate = false, .writeOnly = false}); + + if (!tempFd) + throw NativeSysError("failed to create temporary file %s", PathFmt(tempPath)); + + /* Copy the contents into the created copy. */ + FdSink fileSink(tempFd.get()); + file2.readFile(fileSink); + fileSink.flush(); + args.push_back(tempPath); + + return {std::move(args), std::move(tempFd), std::move(autoDel)}; } } // namespace nix diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc new file mode 100644 index 000000000000..c8e2992be7e2 --- /dev/null +++ b/src/libcmd/flake-schemas.cc @@ -0,0 +1,387 @@ +#include "nix/cmd/flake-schemas.hh" +#include "nix/expr/eval-settings.hh" +#include "nix/fetchers/fetch-to-store.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/flake/provenance.hh" + +namespace nix::flake_schemas { + +using namespace eval_cache; +using namespace flake; + +static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state) +{ + auto accessor = make_ref(); + + accessor->setPathDisplay("«builtin-flake-schemas»"); + + accessor->addFile( + CanonPath("flake.nix"), +#include "builtin-flake-schemas.nix.gen.hh" + ); + + auto [storePath, narHash] = state.store->computeStorePath("source", {accessor}); + + state.allowPath(storePath); // FIXME: should just whitelist the entire virtual store + + state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor); + + // Construct a dummy flakeref. + auto flakeRef = parseFlakeRef( + fetchSettings, + fmt("tarball+https://builtin-flake-schemas?narHash=%s", narHash.to_string(HashFormat::SRI, true))); + + auto flake = readFlake(state, flakeRef, flakeRef, flakeRef, state.storePath(storePath), {}); + + return lockFlake(flakeSettings, state, flakeRef, {}, flake); +} + +ref call( + EvalState & state, + std::shared_ptr lockedFlake, + std::optional defaultSchemasFlake, + bool allowEvalCache) +{ + auto fingerprint = lockedFlake->getFingerprint(*state.store, state.fetchSettings); + + std::string callFlakeSchemasNix = +#include "call-flake-schemas.nix.gen.hh" + ; + + auto lockedDefaultSchemasFlake = defaultSchemasFlake + ? flake::lockFlake(flakeSettings, state, *defaultSchemasFlake, {}) + : getBuiltinDefaultSchemasFlake(state); + auto lockedDefaultSchemasFlakeFingerprint = + lockedDefaultSchemasFlake.getFingerprint(*state.store, state.fetchSettings); + + std::optional fingerprint2; + if (allowEvalCache && evalSettings.useEvalCache && evalSettings.pureEval && fingerprint + && lockedDefaultSchemasFlakeFingerprint) + fingerprint2 = hashString( + HashAlgorithm::SHA256, + fmt("app:%s:%s:%s", + hashString(HashAlgorithm::SHA256, callFlakeSchemasNix).to_string(HashFormat::Base16, false), + fingerprint->to_string(HashFormat::Base16, false), + lockedDefaultSchemasFlakeFingerprint->to_string(HashFormat::Base16, false))); + + if (fingerprint2) { + auto i = state.evalCaches.find(*fingerprint2); + if (i != state.evalCaches.end()) + return i->second; + } + + auto cache = make_ref( + fingerprint2, state, [&state, lockedFlake, callFlakeSchemasNix, lockedDefaultSchemasFlake]() { + auto vCallFlakeSchemas = state.allocValue(); + state.eval( + state.parseExprFromString(callFlakeSchemasNix, state.rootPath(CanonPath::root)), *vCallFlakeSchemas); + + auto vFlake = state.allocValue(); + flake::callFlake(state, *lockedFlake, *vFlake); + + auto vDefaultSchemasFlake = state.allocValue(); + if (vFlake->type() == nAttrs && vFlake->attrs()->get(state.symbols.create("schemas"))) + vDefaultSchemasFlake->mkNull(); + else + flake::callFlake(state, lockedDefaultSchemasFlake, *vDefaultSchemasFlake); + + auto vRes = state.allocValue(); + Value * args[] = {vDefaultSchemasFlake, vFlake}; + state.callFunction(*vCallFlakeSchemas, args, *vRes, noPos); + + return vRes; + }); + + /* Derive the flake output attribute path from the cursor used to + traverse the inventory. We do this so we don't have to maintain + a separate attrpath for that. */ + cache->cleanupAttrPath = [&](AttrPath && attrPath) { + AttrPath res; + auto i = attrPath.begin(); + if (i == attrPath.end()) + return attrPath; + + if (state.symbols[*i] == "inventory") { + ++i; + if (i != attrPath.end()) { + res.push_back(*i++); // copy output name + if (i != attrPath.end()) + ++i; // skip "outputs" + while (i != attrPath.end()) { + ++i; // skip "children" + if (i != attrPath.end()) + res.push_back(*i++); + } + } + } + + else if (state.symbols[*i] == "outputs") { + res.insert(res.begin(), ++i, attrPath.end()); + } + + else + abort(); + + return res; + }; + + if (fingerprint2) + state.evalCaches.emplace(*fingerprint2, cache); + + return cache; +} + +void forEachOutput( + ref inventory, + std::function output, const std::string & doc, bool isLast)> f) +{ + auto outputNames = inventory->getAttrs(); + + auto doOutputs = [&](bool allowIFD) { + evalSettings.enableImportFromDerivation.setDefault(allowIFD); + for (const auto & [i, outputName] : enumerate(outputNames)) { + auto outputInfo = inventory->getAttr(outputName); + try { + auto allowIFDAttr = outputInfo->maybeGetAttr("allowIFD"); + if (allowIFD != (!allowIFDAttr || allowIFDAttr->getBool())) + continue; + auto isUnknown = (bool) outputInfo->maybeGetAttr("unknown"); + auto output = outputInfo->maybeGetAttr("output"); + if (!output && !isUnknown) + // We have a schema but no corresponding output, so skip this. + continue; + Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", outputInfo->getAttrPathStr())); + f(outputName, + isUnknown ? std::shared_ptr() : output, + isUnknown ? "" : outputInfo->getAttr("doc")->getString(), + i + 1 == outputNames.size()); + } catch (Error & e) { + e.addTrace(nullptr, "while evaluating the flake output '%s':", outputInfo->getAttrPathStr()); + throw; + } + } + }; + + // Do outputs that disallow import-from-derivation first. That way, they can't depend on outputs that do allow it. + doOutputs(false); + doOutputs(true); +} + +void visit( + std::optional system, + bool includeLegacy, + ref node, + std::shared_ptr provenance, + std::function visitLeaf, + std::function)> visitNonLeaf, + std::function node, const std::vector & systems)> visitFiltered, + std::function node)> visitLegacy) +{ + Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", node->getAttrPathStr())); + + PushProvenance pushedProvenance( + node->root->state, + provenance ? std::make_shared(provenance, node->getAttrPathStr(), evalSettings.pureEval) + : nullptr); + + /* Filter out legacy outputs, unless --legacy is enabled. */ + if (!includeLegacy) { + if (auto b = node->maybeGetAttr("isLegacy"); b && b->getBool()) { + visitLegacy(node); + return; + } + } + + /* Apply the system type filter. */ + if (system) { + if (auto forSystems = Node(node).forSystems()) { + if (std::find(forSystems->begin(), forSystems->end(), *system) == forSystems->end()) { + visitFiltered(node, *forSystems); + return; + } + } + } + + if (auto children = node->maybeGetAttr("children")) { + visitNonLeaf([&](ForEachChild f) { + auto attrNames = children->getAttrs(); + for (const auto & [i, attrName] : enumerate(attrNames)) { + try { + f(attrName, children->getAttr(attrName), i + 1 == attrNames.size()); + } catch (Error & e) { + // FIXME: use the `isLegacy` attribute. + if (node->root->state.symbols[node->getAttrPath()[0]] != "legacyPackages") { + e.addTrace( + nullptr, "while evaluating the flake output attribute '%s':", node->getAttrPathStr()); + throw; + } + } + } + }); + } + + else + visitLeaf(Leaf(node)); +} + +std::optional> Node::forSystems() const +{ + if (auto forSystems = node->maybeGetAttr("forSystems")) + return forSystems->getListOfStrings(); + else + return std::nullopt; +} + +ref Node::getOutput(const ref & outputs) const +{ + auto res = outputs->findAlongAttrPath(node->getAttrPath()); + if (!res) + throw Error("flake output '%s' should exist according to its schema, but it doesn't", node->getAttrPathStr()); + return *res; +} + +std::optional Leaf::what() const +{ + if (auto what = node->maybeGetAttr("what")) + return what->getString(); + else + return std::nullopt; +} + +std::optional Leaf::shortDescription() const +{ + if (auto what = node->maybeGetAttr("shortDescription")) + return what->getString(); + return std::nullopt; +} + +std::optional Leaf::derivationAttrPath() const +{ + auto n = node->maybeGetAttr("derivationAttrPath"); + if (!n) + return std::nullopt; + return AttrPath::fromStrings(node->root->state, n->getListOfStrings()); +} + +std::shared_ptr Leaf::derivation(const ref & outputs) const +{ + auto path = derivationAttrPath(); + if (!path) { + auto n = node->maybeGetAttr("derivation"); + if (n) + warn( + "Flake output '%s' has a schema that uses the deprecated 'derivation' attribute instead of 'derivationAttrPath'. " + "Please update the schema to use 'derivationAttrPath' instead. " + "You may want to upgrade to version 0.3.0 or higher of https://github.com/DeterminateSystems/flake-schemas.", + node->getAttrPathStr()); + return n; + } + auto drv = getOutput(outputs)->findAlongAttrPath(*path); + if (!drv) + throw Error( + "flake output '%s' does not have a derivation attribute '%s'", + node->getAttrPathStr(), + path->to_string(node->root->state)); + return *drv; +} + +bool Leaf::isFlakeCheck() const +{ + auto isFlakeCheck = node->maybeGetAttr("isFlakeCheck"); + return isFlakeCheck && isFlakeCheck->getBool(); +} + +std::optional getOutputInfo(ref inventory, AttrPath attrPath) +{ + if (attrPath.empty()) + return std::nullopt; + + auto outputName = attrPath.front(); + + auto schemaInfo = inventory->maybeGetAttr(outputName); + if (!schemaInfo) + return std::nullopt; + + auto node = schemaInfo->maybeGetAttr("output"); + if (!node) + return std::nullopt; + + auto pathLeft = std::span(attrPath).subspan(1); + + while (!pathLeft.empty()) { + auto children = node->maybeGetAttr("children"); + if (!children) + break; + auto attr = pathLeft.front(); + node = children->maybeGetAttr(attr); + if (!node) + return std::nullopt; + pathLeft = pathLeft.subspan(1); + } + + return OutputInfo{ + .schemaInfo = ref(schemaInfo), + .nodeInfo = ref(node), + .leafAttrPath = AttrPath(pathLeft.begin(), pathLeft.end()), + }; +} + +Schemas getSchemas(ref inventory) +{ + auto & state(inventory->root->state); + + Schemas schemas; + + for (auto & schemaName : inventory->getAttrs()) { + auto schema = inventory->getAttr(schemaName); + + SchemaInfo schemaInfo; + + if (auto roles = schema->maybeGetAttr("roles")) { + for (auto & roleName : roles->getAttrs()) { + schemaInfo.roles.insert(std::string(state.symbols[roleName])); + } + } + + if (auto appendSystem = schema->maybeGetAttr("appendSystem")) + schemaInfo.appendSystem = appendSystem->getBool(); + + if (auto defaultAttrPath = schema->maybeGetAttr("defaultAttrPath")) { + AttrPath attrPath; + for (auto & s : defaultAttrPath->getListOfStrings()) + attrPath.push_back(state.symbols.create(s)); + schemaInfo.defaultAttrPath = std::move(attrPath); + } + + schemas.insert_or_assign(std::string(state.symbols[schemaName]), std::move(schemaInfo)); + } + + return schemas; +} + +} // namespace nix::flake_schemas + +namespace nix { + +MixFlakeSchemas::MixFlakeSchemas() +{ + addFlag( + {.longName = "default-flake-schemas", + .description = "The URL of the flake providing default flake schema definitions.", + .labels = {"flake-ref"}, + .handler = {&defaultFlakeSchemas}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }}}); +} + +std::optional MixFlakeSchemas::getDefaultFlakeSchemas() +{ + if (!defaultFlakeSchemas) + return std::nullopt; + else + return parseFlakeRef(fetchSettings, *defaultFlakeSchemas, absPath(getCommandBaseDir())); +} + +} // namespace nix diff --git a/src/libcmd/include/nix/cmd/built-path.hh b/src/libcmd/include/nix/cmd/built-path.hh index d41529e5ac4f..34761ee02dc8 100644 --- a/src/libcmd/include/nix/cmd/built-path.hh +++ b/src/libcmd/include/nix/cmd/built-path.hh @@ -105,4 +105,10 @@ struct BuiltPath : _BuiltPathRaw typedef std::vector BuiltPaths; +SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b); + +struct KeyedBuildResult; + +BuiltPath toBuiltPath(KeyedBuildResult & result, ref evalStore, ref store); + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index fc67a60b5b8a..c43b4f87ed43 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -148,7 +148,16 @@ struct MixFlakeOptions : virtual Args, EvalCommand } }; -struct SourceExprCommand : virtual Args, MixFlakeOptions +struct MixFlakeSchemas : virtual Args, virtual StoreCommand +{ + std::optional defaultFlakeSchemas; + + MixFlakeSchemas(); + + std::optional getDefaultFlakeSchemas(); +}; + +struct SourceExprCommand : virtual Args, MixFlakeOptions, MixFlakeSchemas { std::optional file; std::optional expr; @@ -159,9 +168,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions ref parseInstallable(ref store, const std::string & installable); - virtual Strings getDefaultFlakeAttrPaths(); - - virtual Strings getDefaultFlakeAttrPathPrefixes(); + /** + * Return a set of "roles" that this command implements + * (e.g. `nix-build` or `nix-develop`). This is used by flake + * schemas to determine which flake outputs are used as default + * attrpath prefixes. + */ + virtual StringSet getRoles(); /** * Complete an installable from the given prefix. @@ -230,6 +243,8 @@ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); + virtual void preRun(ref store); + virtual void run(ref store, ref installable) = 0; void run(ref store) override; @@ -388,8 +403,7 @@ void completeFlakeRefWithFragment( AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, - Strings attrPathPrefixes, - const Strings & defaultFlakeAttrPaths, + const StringSet & roles, std::string_view prefix); std::string showVersions(const StringSet & versions); @@ -407,13 +421,16 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu struct MixOutLinkBase : virtual Args { /** Prefix for any output symlinks. Empty means do not write an output symlink. */ - std::filesystem::path outLink; + std::optional outLink = std::nullopt; - MixOutLinkBase(const std::string & defaultOutLink) + MixOutLinkBase(const std::optional & defaultOutLink) : outLink(defaultOutLink) { } + /** underlying function */ + void createOutLinksMaybe(const BuiltPaths & paths, ref & store); + /** smaller wrapper for convenience (historically this was the only one) */ void createOutLinksMaybe(const std::vector & buildables, ref & store); }; @@ -435,9 +452,25 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args addFlag({ .longName = "no-link", .description = "Do not create symlinks to the build results.", - .handler = {&outLink, std::filesystem::path{}}, + .handler = {[&] { outLink = std::nullopt; }}, }); } }; +struct MixPrintOutPaths : virtual Args +{ + bool printOutputPaths = false; + + MixPrintOutPaths() + { + addFlag({ + .longName = "print-out-paths", + .description = "Print the resulting output paths", + .handler = {&printOutputPaths, true}, + }); + } + + void printOutPathsMaybe(const BuiltPaths & paths, ref store); +}; + } // namespace nix diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 14897158ae6d..93ffbd435ef9 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -26,9 +26,6 @@ namespace flake { struct Settings; } -/** - * @todo Get rid of global settings variables - */ extern fetchers::Settings fetchSettings; /** diff --git a/src/libcmd/include/nix/cmd/editor-for.hh b/src/libcmd/include/nix/cmd/editor-for.hh index 3fb8a072e732..998b60f4ee25 100644 --- a/src/libcmd/include/nix/cmd/editor-for.hh +++ b/src/libcmd/include/nix/cmd/editor-for.hh @@ -1,15 +1,24 @@ #pragma once ///@file -#include "nix/util/types.hh" #include "nix/util/source-path.hh" +#include "nix/util/os-string.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { /** * Helper function to generate args that invoke $EDITOR on * filename:lineno. + * + * When file doesn't have a physical path, the contents get copied into a + * temporary file, and a file descriptor and RAII cleanup guard for it are + * returned. + * + * @param readOnly make the temporary file readonly if the file has no physical + * path. Ignored otherwise. */ -Strings editorFor(const SourcePath & file, uint32_t line); +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly = true); } // namespace nix diff --git a/src/libcmd/include/nix/cmd/flake-schemas.hh b/src/libcmd/include/nix/cmd/flake-schemas.hh new file mode 100644 index 000000000000..bfc2a38cdef0 --- /dev/null +++ b/src/libcmd/include/nix/cmd/flake-schemas.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "nix/expr/eval-cache.hh" +#include "nix/flake/flake.hh" +#include "nix/cmd/command.hh" + +namespace nix::flake_schemas { + +using namespace eval_cache; + +ref call( + EvalState & state, + std::shared_ptr lockedFlake, + std::optional defaultSchemasFlake, + bool allowEvalCache = true); + +void forEachOutput( + ref inventory, + std::function output, const std::string & doc, bool isLast)> f); + +/** + * A convenience wrapper around `AttrCursor` for nodes in the `inventory` tree returned by call-flake-schemas.nix. + */ +struct Node +{ + const ref node; + + Node(const ref & node) + : node(node) + { + } + + /** + * Return the `forSystems` attribute. This can be null, which + * means "all systems". + */ + std::optional> forSystems() const; + + /** + * Return the actual output corresponding to this info node. + */ + ref getOutput(const ref & outputs) const; +}; + +struct Leaf : Node +{ + using Node::Node; + + std::optional what() const; + + std::optional shortDescription() const; + + std::optional derivationAttrPath() const; + + /** + * Return the attribute corresponding to `derivationAttrPath`, if set. + */ + std::shared_ptr derivation(const ref & outputs) const; + + bool isFlakeCheck() const; +}; + +typedef std::function attr, bool isLast)> ForEachChild; + +void visit( + std::optional system, + bool includeLegacy, + ref node, + std::shared_ptr provenance, + std::function visitLeaf, + std::function)> visitNonLeaf, + std::function node, const std::vector & systems)> visitFiltered, + std::function node)> visitLegacy); + +struct OutputInfo +{ + ref schemaInfo; + ref nodeInfo; + AttrPath leafAttrPath; +}; + +std::optional getOutputInfo(ref inventory, AttrPath attrPath); + +struct SchemaInfo +{ + std::string doc; + StringSet roles; + bool appendSystem = false; + std::optional defaultAttrPath; +}; + +using Schemas = std::map; + +Schemas getSchemas(ref root); + +} // namespace nix::flake_schemas diff --git a/src/libcmd/include/nix/cmd/installable-attr-path.hh b/src/libcmd/include/nix/cmd/installable-attr-path.hh index 474bb358ec91..c33b119a5fa8 100644 --- a/src/libcmd/include/nix/cmd/installable-attr-path.hh +++ b/src/libcmd/include/nix/cmd/installable-attr-path.hh @@ -17,12 +17,11 @@ #include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" +#include "nix/expr/root-value.hh" #include #include -#include - namespace nix { class InstallableAttrPath : public InstallableValue diff --git a/src/libcmd/include/nix/cmd/installable-flake.hh b/src/libcmd/include/nix/cmd/installable-flake.hh index 9f449ad48f2e..3acce913dcb7 100644 --- a/src/libcmd/include/nix/cmd/installable-flake.hh +++ b/src/libcmd/include/nix/cmd/installable-flake.hh @@ -36,11 +36,14 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue struct InstallableFlake : InstallableValue { FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; + std::string fragment; + AttrPath parsedFragment; + StringSet roles; ExtendedOutputsSpec extendedOutputsSpec; const flake::LockFlags & lockFlags; mutable std::shared_ptr _lockedFlake; + bool useEvalCache = true; + std::optional defaultFlakeSchemas; InstallableFlake( SourceExprCommand * cmd, @@ -48,17 +51,15 @@ struct InstallableFlake : InstallableValue FlakeRef && flakeRef, std::string_view fragment, ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); + StringSet roles, + const flake::LockFlags & lockFlags, + std::optional defaultFlakeSchemas); std::string what() const override { - return flakeRef.to_string() + "#" + *attrPaths.begin(); + return flakeRef.to_string() + "#" + fragment; } - std::vector getActualAttrPaths(); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -67,11 +68,23 @@ struct InstallableFlake : InstallableValue * Get a cursor to every attrpath in getActualAttrPaths() that * exists. However if none exists, throw an exception. */ - std::vector> getCursors(EvalState & state) override; + std::vector> getCursors(EvalState & state, bool useDefaultAttrPath) override; + + void getCompletions(const std::string & flakeRefS, AddCompletions & completions); ref getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const; + + std::shared_ptr makeProvenance(std::string_view attrPath) const; + + ref openEvalCache() const; + +private: + + mutable std::shared_ptr _evalCache; + + std::vector getAttrPaths(bool useDefaultAttrPath, ref inventory); }; /** diff --git a/src/libcmd/include/nix/cmd/installable-value.hh b/src/libcmd/include/nix/cmd/installable-value.hh index 27a1fb9815d4..09178c96c972 100644 --- a/src/libcmd/include/nix/cmd/installable-value.hh +++ b/src/libcmd/include/nix/cmd/installable-value.hh @@ -93,7 +93,7 @@ struct InstallableValue : Installable * However if none exists, throw exception instead of returning * empty vector. */ - virtual std::vector> getCursors(EvalState & state); + virtual std::vector> getCursors(EvalState & state, bool useDefaultAttrPath = true); /** * Get the first and most preferred cursor this Installable could diff --git a/src/libcmd/include/nix/cmd/installables.hh b/src/libcmd/include/nix/cmd/installables.hh index 530334e037b7..2ea35261c7fa 100644 --- a/src/libcmd/include/nix/cmd/installables.hh +++ b/src/libcmd/include/nix/cmd/installables.hh @@ -96,6 +96,22 @@ typedef std::vector DerivedPathsWithInfo; struct Installable; +struct InstallableWithBuildResult +{ + ref installable; + + using Success = BuiltPathWithResult; + + using Failure = BuildResult; // must be a `BuildResult::Failure` + + std::variant result; + + /** + * Throw an exception if this represents a failure, otherwise returns a `BuiltPathWithResult`. + */ + const BuiltPathWithResult & getSuccess() const; +}; + /** * Shorthand, for less typing and helping us keep the choice of * collection in sync. @@ -160,13 +176,15 @@ struct Installable const Installables & installables, BuildMode bMode = bmNormal); - static std::vector, BuiltPathWithResult>> build2( + static std::vector build2( ref evalStore, ref store, Realise mode, const Installables & installables, BuildMode bMode = bmNormal); + static void throwBuildErrors(std::vector & buildResults, const Store & store); + static std::set toStorePathSet( ref evalStore, ref store, Realise mode, OperateOn operateOn, const Installables & installables); diff --git a/src/libcmd/include/nix/cmd/markdown.hh b/src/libcmd/include/nix/cmd/markdown.hh index 95a59c2aa7b9..716b87b87eef 100644 --- a/src/libcmd/include/nix/cmd/markdown.hh +++ b/src/libcmd/include/nix/cmd/markdown.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include namespace nix { diff --git a/src/libcmd/include/nix/cmd/meson.build b/src/libcmd/include/nix/cmd/meson.build index ac074b06ff3c..6dea1ea5be8c 100644 --- a/src/libcmd/include/nix/cmd/meson.build +++ b/src/libcmd/include/nix/cmd/meson.build @@ -9,6 +9,7 @@ headers = files( 'common-eval-args.hh', 'compatibility-settings.hh', 'editor-for.hh', + 'flake-schemas.hh', 'get-build-log.hh', 'installable-attr-path.hh', 'installable-derived-path.hh', diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e139076802..5c38a2593427 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/util/args.hh" #include "nix/store/content-address.hh" diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059c9..9b9a03c3e423 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index d46aa94b6b50..81c7b8df5a2d 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -28,7 +28,7 @@ struct AbstractNixRepl * @param programName Name of the command, e.g. `nix` or `nix-env`. * @param args arguments to the command. */ - using RunNix = void(const std::string & programName, OsStrings args, const std::optional & input); + using RunNix = void(const std::string & programName, OsStrings args); /** * @param runNix Function to run the nix CLI to support various diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 3aeee5b55e1c..e01475104b5a 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -53,8 +53,30 @@ struct ServeUnixSocketOptions * Mode for the created socket file. */ mode_t socketMode = 0666; + +#ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + + /** + * Additional file descriptor to poll. Useful for doing a self-pipe trick + * https://cr.yp.to/docs/selfpipe.html. + */ + Descriptor auxiliaryFd = INVALID_DESCRIPTOR; + + /** + * Optional callback invoked on POLLIN event for auxiliaryFd. + */ + std::function onAuxiliaryFdPollin = nullptr; +#endif }; +MakeError(AbortServeSocket, BaseError); + /** * Run a server loop that accepts connections and calls the handler for each. * @@ -70,6 +92,7 @@ struct ServeUnixSocketOptions * * This function never returns normally. It runs until interrupted * (e.g., via SIGINT), at which point it throws `Interrupted`. + * Can be explicitly exited by throwing AbortServeSocket. * * @param options Configuration for the server. * @param handler Callback invoked for each accepted connection. diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 28c3db3fc79a..524dc799f02f 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,24 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-attr-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" -#include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include @@ -32,7 +21,7 @@ InstallableAttrPath::InstallableAttrPath( ExtendedOutputsSpec extendedOutputsSpec) : InstallableValue(state) , cmd(cmd) - , v(allocRootValue(v)) + , v(RootValue(v)) , attrPath(attrPath) , extendedOutputsSpec(std::move(extendedOutputsSpec)) { @@ -89,7 +78,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() } DerivedPathsWithInfo res; - for (auto & [drvPath, outputs] : byDrvPath) + for (auto & [drvPath, outputs] : byDrvPath) { + state->waitForPath(drvPath); res.push_back({ .path = DerivedPath::Built{ @@ -102,6 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() so we can fill in this info. */ }), }); + } return res; } diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 77a7c8d6ec1b..d2022aeb09d0 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -1,56 +1,30 @@ #include "nix/store/globals.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" -#include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" +#include "nix/expr/eval-error.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include +#include "nix/flake/provenance.hh" +#include "nix/cmd/flake-schemas.hh" #include namespace nix { -std::vector InstallableFlake::getActualAttrPaths() -{ - std::vector res; - if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")) { - attrPaths.front().erase(0, 1); - res.push_back(attrPaths.front()); - return res; - } - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -static std::string showAttrPaths(const std::vector & paths) +static std::string showAttrPaths(EvalState & state, const std::vector & paths) { std::string s; for (const auto & [n, i] : enumerate(paths)) { if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; s += '\''; - s += i; + s += i.to_string(state); s += '\''; } return s; @@ -62,15 +36,17 @@ InstallableFlake::InstallableFlake( FlakeRef && flakeRef, std::string_view fragment, ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) + StringSet roles, + const flake::LockFlags & lockFlags, + std::optional defaultFlakeSchemas) : InstallableValue(state) , flakeRef(flakeRef) - , attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}) - , prefixes(fragment == "" ? Strings{} : prefixes) + , fragment(fragment) + , parsedFragment(AttrPath::parse(*state, fragment)) + , roles(roles) , extendedOutputsSpec(std::move(extendedOutputsSpec)) , lockFlags(lockFlags) + , defaultFlakeSchemas(defaultFlakeSchemas) { if (cmd && cmd->getAutoArgs(*state)->size()) throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); @@ -84,6 +60,8 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto attrPath = attr->getAttrPathStr(); + PushProvenance pushedProvenance(*state, makeProvenance(attrPath)); + if (!attr->isDerivation()) { // FIXME: use eval cache? @@ -102,6 +80,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } auto drvPath = attr->forceDerivation(); + state->waitForPath(drvPath); std::optional priority; @@ -157,34 +136,174 @@ std::pair InstallableFlake::toValue(EvalState & state) return {&getCursor(state)->forceValue(), noPos}; } -std::vector> InstallableFlake::getCursors(EvalState & state) +std::vector InstallableFlake::getAttrPaths(bool useDefaultAttrPath, ref inventory) +{ + if (fragment.starts_with(".")) + return {AttrPath::parse(*state, fragment.substr(1))}; + + std::vector attrPaths; + + auto schemas = flake_schemas::getSchemas(inventory); + + // FIXME: Ugly hack to preserve the historical precedence + // between outputs. We should add a way for schemas to declare + // priorities. + std::vector schemasSorted; + std::set schemasSeen; + auto doSchema = [&](const std::string & schema) { + if (schemas.contains(schema)) { + schemasSorted.push_back(schema); + schemasSeen.insert(schema); + } + }; + doSchema("apps"); + doSchema("defaultApp"); + doSchema("devShells"); + doSchema("devShell"); + doSchema("packages"); + doSchema("defaultPackage"); + doSchema("legacyPackages"); + for (auto & schema : schemas) + if (!schemasSeen.contains(schema.first)) + schemasSorted.push_back(schema.first); + + for (auto & role : roles) { + for (auto & schemaName : schemasSorted) { + auto & schema = schemas.find(schemaName)->second; + if (schema.roles.contains(role)) { + AttrPath attrPath{state->symbols.create(schemaName)}; + if (schema.appendSystem) + attrPath.push_back(state->symbols.create(settings.thisSystem.get())); + + if (useDefaultAttrPath && parsedFragment.empty()) { + if (schema.defaultAttrPath) { + auto attrPath2{attrPath}; + for (auto & x : *schema.defaultAttrPath) + attrPath2.push_back(x); + attrPaths.push_back(attrPath2); + } + } else { + auto attrPath2{attrPath}; + for (auto & x : parsedFragment) + attrPath2.push_back(x); + attrPaths.push_back(attrPath2); + } + } + } + } + + if (!parsedFragment.empty()) + attrPaths.push_back(parsedFragment); + + // FIXME: compatibility hack to get `nix repl` to return all + // outputs by default. + if (parsedFragment.empty() && roles.contains("nix-repl")) + attrPaths.push_back({}); + + return attrPaths; +} + +std::vector> InstallableFlake::getCursors(EvalState & state, bool useDefaultAttrPath) { - auto evalCache = openEvalCache(state, getLockedFlake()); + auto cache = openEvalCache(); - auto root = evalCache->getRoot(); + auto inventory = cache->getRoot()->getAttr("inventory"); + auto outputs = cache->getRoot()->getAttr("outputs"); + + auto attrPaths = getAttrPaths(useDefaultAttrPath, inventory); + + if (attrPaths.empty()) + throw Error( + "Flake '%s' does not have any schema that provides a default output for the role(s) %s.", + flakeRef, + concatStringsSep(", ", roles)); std::vector> res; Suggestions suggestions; - auto attrPaths = getActualAttrPaths(); for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); + debug("trying flake output attribute '%s'", attrPath.to_string(state)); - auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath)); - if (attr) { - res.push_back(ref(*attr)); - } else { - suggestions += attr.getSuggestions(); + PushProvenance pushedProvenance(state, makeProvenance(attrPath.to_string(state))); + +#if 0 + auto outputInfo = flake_schemas::getOutputInfo(inventory, attrPath); + + if (outputInfo && outputInfo->leafAttrPath.empty()) { + if (auto drv = outputInfo->nodeInfo->maybeGetAttr("derivation")) { + res.push_back(ref(drv)); + continue; + } + } +#endif + + try { + auto attr = outputs->findAlongAttrPath(attrPath); + if (attr) + res.push_back(ref(*attr)); + else + suggestions += attr.getSuggestions(); + } catch (TypeError & e) { + debug("error resolving attribute '%s': %s", attrPath.to_string(state), e.msg()); + // Continue to next attribute path } } if (res.size() == 0) - throw Error(suggestions, "flake '%s' does not provide attribute %s", flakeRef, showAttrPaths(attrPaths)); + throw Error(suggestions, "flake '%s' does not provide attribute %s", flakeRef, showAttrPaths(state, attrPaths)); return res; } +void InstallableFlake::getCompletions(const std::string & flakeRefS, AddCompletions & completions) +{ + auto cache = openEvalCache(); + + auto inventory = cache->getRoot()->getAttr("inventory"); + auto outputs = cache->getRoot()->getAttr("outputs"); + + if (fragment.ends_with(".") || fragment.empty()) + // Represent that we're looking for attributes starting with the empty prefix (i.e. all attributes inside the + // parent. + parsedFragment.push_back(state->symbols.create("")); + + auto attrPaths = getAttrPaths(true, inventory); + + if (fragment.empty()) + // Return all top-level flake outputs. + attrPaths.push_back(AttrPath{state->symbols.create("")}); + + auto lastAttr = fragment.ends_with(".") || parsedFragment.empty() ? std::string_view("") + : state->symbols[parsedFragment.back()]; + std::string prefix; + if (auto dot = fragment.rfind('.'); dot != std::string::npos) + prefix = fragment.substr(0, dot); + if (fragment.starts_with(".") && !prefix.starts_with(".")) + prefix = "." + prefix; + + for (auto attrPath : attrPaths) { + if (attrPath.empty()) + attrPath.push_back(state->symbols.create("")); + + auto attrPathParent{attrPath}; + attrPathParent.pop_back(); + + auto attr = outputs->findAlongAttrPath(attrPathParent); + if (!attr) + continue; + + for (auto & childName : (*attr)->getAttrs()) { + if (hasPrefix(state->symbols[childName], lastAttr)) { + auto attrPathChild = (*attr)->getAttrPath(childName); + completions.add( + flakeRefS + "#" + prefix + (prefix.empty() || prefix.ends_with(".") ? "" : ".") + + state->symbols[childName]); + } + } + } +} + ref InstallableFlake::getLockedFlake() const { if (!_lockedFlake) { @@ -197,6 +316,14 @@ ref InstallableFlake::getLockedFlake() const return ref(_lockedFlake); } +ref InstallableFlake::openEvalCache() const +{ + if (!_evalCache) { + _evalCache = flake_schemas::call(*state, getLockedFlake(), defaultFlakeSchemas, useEvalCache); + } + return ref(_evalCache); +} + FlakeRef InstallableFlake::nixpkgsFlakeRef() const { auto lockedFlake = getLockedFlake(); @@ -213,4 +340,12 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const return defaultNixpkgsFlakeRef(); } +std::shared_ptr InstallableFlake::makeProvenance(std::string_view attrPath) const +{ + auto provenance = getLockedFlake()->flake.provenance; + if (!provenance) + return nullptr; + return std::make_shared(provenance, std::string(attrPath), evalSettings.pureEval); +} + } // namespace nix diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db49..6c2fd60efd8e 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -4,7 +4,7 @@ namespace nix { -std::vector> InstallableValue::getCursors(EvalState & state) +std::vector> InstallableValue::getCursors(EvalState & state, bool useDefaultAttrPath) { auto evalCache = std::make_shared(std::nullopt, state, [&]() { return toValue(state).first; }); @@ -15,7 +15,7 @@ ref InstallableValue::getCursor(EvalState & state) { /* Although getCursors should return at least one element, in case it doesn't, bound check to avoid an undefined behavior for vector[0] */ - return getCursors(state).at(0); + return getCursors(state, true).at(0); } static UsageError nonValueInstallable(Installable & installable) @@ -55,7 +55,7 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: else if (v.type() == nString) { return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(state->devirtualize(state->coerceToSingleDerivedPath(pos, v, errorCtx))), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 8f8309bd96d7..2285e0c3cf51 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,3 +1,4 @@ +#include "nix/cmd/built-path.hh" #include "nix/store/globals.hh" #include "nix/cmd/installables.hh" #include "nix/cmd/installable-derived-path.hh" @@ -13,17 +14,13 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" -#include "nix/expr/get-drvs.hh" #include "nix/store/store-api.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" - -#include -#include +#include "nix/util/exit.hh" #include @@ -241,19 +238,9 @@ MixReadOnlyOption::MixReadOnlyOption() }); } -Strings SourceExprCommand::getDefaultFlakeAttrPaths() +StringSet SourceExprCommand::getRoles() { - return {"packages." + settings.thisSystem.get() + ".default", "defaultPackage." + settings.thisSystem.get()}; -} - -Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() -{ - return {// As a convenience, look for the attribute in - // 'outputs.packages'. - "packages." + settings.thisSystem.get() + ".", - // As a temporary hack until Nixpkgs is properly converted - // to provide a clean 'packages' set, look in 'legacyPackages'. - "legacyPackages." + settings.thisSystem.get() + "."}; + return {"nix-build"}; } Args::CompleterClosure SourceExprCommand::getCompleteInstallable() @@ -307,13 +294,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s } } } else { - completeFlakeRefWithFragment( - completions, - getEvalState(), - lockFlags, - getDefaultFlakeAttrPathPrefixes(), - getDefaultFlakeAttrPaths(), - prefix); + completeFlakeRefWithFragment(completions, getEvalState(), lockFlags, getRoles(), prefix); } } catch (EvalError &) { // Don't want eval errors to mess-up with the completion engine, so let's just swallow them @@ -324,91 +305,37 @@ void completeFlakeRefWithFragment( AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, - Strings attrPathPrefixes, - const Strings & defaultFlakeAttrPaths, + const StringSet & roles, std::string_view prefix) -{ - /* Look for flake output attributes that match the - prefix. */ - try { - auto hash = prefix.find('#'); - if (hash == std::string::npos) { - completeFlakeRef(completions, evalState->store, prefix); - } else { - completions.setType(AddCompletions::Type::Attrs); - - auto fragment = prefix.substr(hash + 1); - std::string prefixRoot = ""; - if (fragment.starts_with(".")) { - fragment = fragment.substr(1); - prefixRoot = "."; - } - auto flakeRefS = std::string(prefix.substr(0, hash)); - - // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = - parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string()); - - auto evalCache = openEvalCache( - *evalState, make_ref(lockFlake(flakeSettings, *evalState, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - if (prefixRoot == ".") { - attrPathPrefixes.clear(); - } - /* Complete 'fragment' relative to all the - attrpath prefixes as well as the root of the - flake. */ - attrPathPrefixes.push_back(""); - - for (auto & attrPathPrefixS : attrPathPrefixes) { - auto attrPathPrefix = AttrPath::parse(*evalState, attrPathPrefixS); - auto attrPathS = attrPathPrefixS + std::string(fragment); - auto attrPath = AttrPath::parse(*evalState, attrPathS); - - std::string lastAttr; - if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) { - lastAttr = evalState->symbols[attrPath.back()]; - attrPath.pop_back(); - } +try { + auto hash = prefix.find('#'); + if (hash == std::string::npos) { + completeFlakeRef(completions, evalState->store, prefix); + return; + } - auto attr = root->findAlongAttrPath(attrPath); - if (!attr) - continue; + completions.setType(AddCompletions::Type::Attrs); - for (auto & attr2 : (*attr)->getAttrs()) { - if (hasPrefix(evalState->symbols[attr2], lastAttr)) { - auto attrPath2 = (*attr)->getAttrPath(attr2); - /* Strip the attrpath prefix. */ - attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - // FIXME: handle names with dots - completions.add(flakeRefS + "#" + prefixRoot + attrPath2.to_string(*evalState)); - } - } - } + auto fragment = prefix.substr(hash + 1); + auto flakeRefS = std::string(prefix.substr(0, hash)); - /* And add an empty completion for the default - attrpaths. */ - if (fragment.empty()) { - for (auto & attrPath : defaultFlakeAttrPaths) { - auto attr = root->findAlongAttrPath(AttrPath::parse(*evalState, attrPath)); - if (!attr) - continue; - completions.add(flakeRefS + "#" + prefixRoot); - } - } - } - } catch (Error & e) { - logWarning(e.info()); - } + InstallableFlake{ + nullptr, + evalState, + // TODO: ideally this would use the command base directory instead of assuming ".". + parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string()), + fragment, + ExtendedOutputsSpec::Default{}, // FIXME: could be that we're completing the outputs spec... + roles, + lockFlags, + {}} + .getCompletions(flakeRefS, completions); +} catch (Error & e) { + logWarning(e.info()); } void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - return; - if (prefix == "") completions.add("."); @@ -518,9 +445,9 @@ Installables SourceExprCommand::parseInstallables(ref store, std::vector< std::move(flakeRef), fragment, std::move(extendedOutputsSpec), - getDefaultFlakeAttrPaths(), - getDefaultFlakeAttrPathPrefixes(), - lockFlags)); + getRoles(), + lockFlags, + getDefaultFlakeSchemas())); continue; } catch (...) { ex = std::current_exception(); @@ -540,64 +467,69 @@ ref SourceExprCommand::parseInstallable(ref store, const std return installables.front(); } -static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { return SingleBuiltPath::Opaque{bo.path}; }, - [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { - auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); - // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. - SingleDerivedPath::Built truncatedBfd{ - .drvPath = makeConstantStorePathRef(drvPath.outPath()), - .output = bfd.output, - }; - auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); - return SingleBuiltPath::Built{ - .drvPath = make_ref(std::move(drvPath)), - .output = {bfd.output, outputPath}, - }; - }, - }, - b.raw()); -} - -std::vector Installable::build( - ref evalStore, ref store, Realise mode, const Installables & installables, BuildMode bMode) +const BuiltPathWithResult & InstallableWithBuildResult::getSuccess() const { - std::vector res; - for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode)) - res.push_back(builtPathWithResult); - return res; + if (auto * failure = std::get_if(&result)) { + auto failure2 = failure->tryGetFailure(); + assert(failure2); + throw *failure2; + } else + return *std::get_if(&result); } -static void throwBuildErrors(std::vector & buildResults, const Store & store) +void Installable::throwBuildErrors(std::vector & buildResults, const Store & store) { - std::vector> failed; for (auto & buildResult : buildResults) { - if (auto * failure = buildResult.tryGetFailure()) { - failed.push_back({&buildResult, failure}); - } - } + if (std::get_if(&buildResult.result)) { + // Report success first. + for (auto & buildResult : buildResults) { + if (std::get_if(&buildResult.result)) + notice("✅ " ANSI_BOLD "%s" ANSI_NORMAL, buildResult.installable->what()); + } - auto failedResult = failed.begin(); - if (failedResult != failed.end()) { - if (failed.size() == 1) { - throw *failedResult->second; - } else { - StringSet failedPaths; - for (; failedResult != failed.end(); failedResult++) { - if (!failedResult->second->message().empty()) { - logError(failedResult->second->info()); + // Then cancelled builds. + for (auto & buildResult : buildResults) { + if (auto failure = std::get_if(&buildResult.result)) { + if (failure->isCancelled()) + notice( + "❓ " ANSI_BOLD "%s" ANSI_NORMAL ANSI_FAINT " (cancelled)", + buildResult.installable->what()); } - failedPaths.insert(failedResult->first->path.to_string(store)); } - throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + + // Then failures. + for (auto & buildResult : buildResults) { + if (auto failure = std::get_if(&buildResult.result)) { + if (failure->isCancelled()) + continue; + auto failure2 = failure->tryGetFailure(); + assert(failure2); + printError("❌ " ANSI_RED "%s" ANSI_NORMAL, buildResult.installable->what()); + try { + throw *failure2; + } catch (Error & e) { + logError(e.info()); + } + } + } + + throw Exit(1); } } } -std::vector, BuiltPathWithResult>> Installable::build2( +std::vector Installable::build( + ref evalStore, ref store, Realise mode, const Installables & installables, BuildMode bMode) +{ + auto results = build2(evalStore, store, mode, installables, bMode); + throwBuildErrors(results, *store); + std::vector res; + for (auto & b : results) + res.push_back(b.getSuccess()); + return res; +} + +std::vector Installable::build2( ref evalStore, ref store, Realise mode, const Installables & installables, BuildMode bMode) { if (mode == Realise::Nothing) @@ -619,7 +551,7 @@ std::vector, BuiltPathWithResult>> Installable::build } } - std::vector, BuiltPathWithResult>> res; + std::vector res; switch (mode) { @@ -634,17 +566,21 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back( - {aux.installable, - {.path = - BuiltPath::Built{ - .drvPath = - make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), - .outputs = outputs, - }, - .info = aux.info}}); + {.installable = aux.installable, + .result = InstallableWithBuildResult::Success{ + .path = + BuiltPath::Built{ + .drvPath = make_ref( + getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({aux.installable, {.path = BuiltPath::Opaque{bo.path}, .info = aux.info}}); + res.push_back( + {.installable = aux.installable, + .result = InstallableWithBuildResult::Success{ + .path = BuiltPath::Opaque{bo.path}, .info = aux.info}}); }, }, path.raw()); @@ -658,35 +594,22 @@ std::vector, BuiltPathWithResult>> Installable::build printMissing(store, pathsToBuild, lvlInfo); auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); - throwBuildErrors(buildResults, *store); for (auto & buildResult : buildResults) { - // If we didn't throw, they must all be sucesses - auto & success = std::get(buildResult.inner); + if (buildResult.tryGetFailure()) { + for (auto & aux : backmap[buildResult.path]) { + res.push_back({.installable = aux.installable, .result = buildResult}); + } + continue; + } for (auto & aux : backmap[buildResult.path]) { - std::visit( - overloaded{ - [&](const DerivedPath::Built & bfd) { - std::map outputs; - for (auto & [outputName, realisation] : success.builtOutputs) - outputs.emplace(outputName, realisation.outPath); - res.push_back( - {aux.installable, - {.path = - BuiltPath::Built{ - .drvPath = - make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), - .outputs = outputs, - }, - .info = aux.info, - .result = buildResult}}); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back( - {aux.installable, - {.path = BuiltPath::Opaque{bo.path}, .info = aux.info, .result = buildResult}}); - }, + res.push_back({ + aux.installable, + BuiltPathWithResult{ + .path = toBuiltPath(buildResult, evalStore, store), + .info = aux.info, + .result = buildResult, }, - buildResult.path.raw()); + }); } } @@ -844,8 +767,11 @@ InstallableCommand::InstallableCommand() }); } +void InstallableCommand::preRun(ref store) {} + void InstallableCommand::run(ref store) { + preRun(store); auto installable = parseInstallable(store, _installable); run(store, std::move(installable)); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index d970a8e4b066..680b8136dd32 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -71,6 +71,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/generate-header') sources = files( 'built-path.cc', @@ -78,6 +79,7 @@ sources = files( 'command.cc', 'common-eval-args.cc', 'editor-for.cc', + 'flake-schemas.cc', 'get-build-log.cc', 'installable-attr-path.cc', 'installable-derived-path.cc', @@ -97,6 +99,11 @@ if host_machine.system() != 'windows' ) endif +sources += [ + gen_header.process('call-flake-schemas.nix'), + gen_header.process('builtin-flake-schemas.nix'), +] + subdir('include/nix/cmd') subdir('nix-meson-build-support/export-all-symbols') @@ -106,11 +113,10 @@ this_library = library( 'nixcmd', sources, config_priv_h, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index c382f0e5760d..1d677142da1d 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -35,7 +35,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-cmd"; + pname = "determinate-nix-cmd"; inherit version; workDir = ./.; @@ -49,6 +49,8 @@ mkMesonLibrary (finalAttrs: { ./include/nix/cmd/meson.build (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) + ./call-flake-schemas.nix + ./builtin-flake-schemas.nix ]; buildInputs = [ diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 8eebeec25b5e..5f2417dc1331 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,11 +18,10 @@ extern "C" { } #endif -#include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/cmd/repl-interacter.hh" #include "nix/util/file-system.hh" -#include "nix/cmd/repl.hh" +#include "nix/util/serialise.hh" #include "nix/util/environment-variables.hh" namespace nix { @@ -123,9 +122,38 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter logWarning(e.info()); } #if !USE_READLINE - el_hist_size = 1000; -#endif + /* editline's read_history uses a fixed 256-byte buffer (SCREEN_INC), + which silently splits lines longer than 255 characters into separate + history entries. Read the file ourselves to avoid the length limit. See: + https://github.com/troglobit/editline/blob/2e0504d31e6878208036a4dd91f44841dabb1ee7/src/editline.c#L1617-L1635 + + ::rl_initialize must be called before the subsequent calls to + ::add_history to ensure that the buffer has actually been allocated + (but before setting ::el_hist_size). + Best I can tell it's supposed to idempotent, e.g. ::readline calls it + unconditionally anyway. */ + + ::el_hist_size = 1000; /* FIXME: Why the arbitrary limit? */ + ::rl_initialize(); + + auto fd = openFileReadonly(historyFile); + if (!fd) { + NativeSysError err("opening file %s", PathFmt(historyFile)); + if (!err.is(std::errc::no_such_file_or_directory) && !err.is(std::errc::not_a_directory)) + logWarning(err.info()); + } else { + try { + FdSource source(fd.get()); + while (true) + add_history(source.readLine().c_str()); + } catch (EndOfFile &) { + } catch (SystemError & e) { + logWarning(e.info()); + } + } +#else read_history(historyFile.string().c_str()); +#endif auto oldRepl = curRepl; curRepl = repl; Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); @@ -175,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -187,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 52ecc9092328..6eaab847a0a6 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -17,6 +17,7 @@ #include "nix/cmd/get-build-log.hh" #include "nix/expr/get-drvs.hh" #include "nix/store/derivations.hh" +#include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" #include "nix/flake/flake.hh" #include "nix/flake/lockfile.hh" @@ -74,7 +75,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc RunNix * runNixPtr; - void runNix(const std::string & program, OsStrings args, const std::optional & input = {}); + void runNix(const std::string & program, OsStrings args); std::unique_ptr interacter; @@ -160,7 +161,24 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } -MakeError(IncompleteReplExpr, ParseError); +/** + * Thrown when the REPL's own input is incomplete (e.g. unclosed multi-line + * string or open parenthesis). The mainLoop catches this to prompt for + * continuation lines instead of showing an error. + * + * Only parseString and parseReplBindings may throw this. Evaluation can also + * produce "unexpected end of file" ParseErrors (e.g. `import ./broken.nix`), + * but those must be reported as errors, not trigger continuation. The + * exception subtype is what distinguishes the two cases. + */ +MakeError(IncompleteReplExpr, Error); + +static bool isIncompleteInput(const ParseError & e) +{ + return e.msg().find("unexpected end of file") != std::string::npos; +} + +void IncompleteReplExpr::anchor() {} static bool isFirstRepl = true; @@ -171,7 +189,7 @@ ReplExitStatus NixRepl::mainLoop() if (state->debugRepl) { debuggerNotice = " debugger"; } - notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice); + notice("Nix %1%%2%\nType :? for help.", version(), debuggerNotice); } isFirstRepl = false; @@ -311,6 +329,7 @@ StorePath NixRepl::getDerivationPath(Value & v) auto drvPath = packageInfo->queryDrvPath(); if (!drvPath) throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); + state->waitForPath(*drvPath); if (!state->store->isValidPath(*drvPath)) throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath)); return *drvPath; @@ -326,7 +345,7 @@ void NixRepl::loadDebugTraceEnv(DebugTrace & dt) // add staticenv vars. for (auto & [name, value] : *(vm.get())) - addVarToScope(state->symbols.create(name), *value); + addVarToScope(state->symbols.create(name), **value); } } @@ -483,8 +502,8 @@ ProcessLineResult NixRepl::processLine(std::string line) } }(); - // Open in EDITOR - auto args = editorFor(path, line); + /* Open file in EDITOR, or edit a read-only copy if the file doesn't have a physical path. */ + auto [args, fd, autoDel] = editorFor(path, line, /*readOnly=*/true); auto editor = args.front(); args.pop_front(); @@ -493,13 +512,16 @@ ProcessLineResult NixRepl::processLine(std::string line) runProgram2({ .program = editor, .lookupPath = true, - .args = toOsStrings(std::move(args)), + .args = std::move(args), .isInteractive = true, }); - // Reload right after exiting the editor - state->resetFileCache(); - reloadFilesAndFlakes(); + /* If we had to open a temporary read-only file, there's no need to + reload (no files could have changed anyway). */ + if (!fd) { + state->resetFileCache(); + reloadFilesAndFlakes(); + } } else if (command == ":t") { @@ -535,7 +557,7 @@ ProcessLineResult NixRepl::processLine(std::string line) }); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { + for (auto & [outputName, outputPath] : deepQueryDerivationOutputMap(*state->store, drvPath)) { auto localStore = state->store.dynamic_pointer_cast(); if (localStore && command == ":bl") { std::string symlink = "repl-result-" + outputName; @@ -656,11 +678,7 @@ ProcessLineResult NixRepl::processLine(std::string line) else { // Try parsing as bindings first (handles `x = 1`, `inherit ...`, etc.) - ExprAttrs * bindings = nullptr; - try { - bindings = parseReplBindings(line); - } catch (ParseError &) { - } + ExprAttrs * bindings = parseReplBindings(line); if (bindings) { Env * inheritEnv = bindings->inheritFromExprs ? bindings->buildInheritFromEnv(*state, *env) : nullptr; @@ -684,12 +702,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -697,9 +716,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -726,6 +742,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -768,28 +788,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } @@ -850,12 +886,9 @@ Expr * NixRepl::parseString(std::string s) try { return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } catch (ParseError & e) { - if (e.msg().find("unexpected end of file") != std::string::npos) - // For parse errors on incomplete input, we continue waiting for the next line of - // input without clearing the input so far. + if (isIncompleteInput(e)) throw IncompleteReplExpr(e.msg()); - else - throw; + throw; } } @@ -864,20 +897,20 @@ ExprAttrs * NixRepl::parseReplBindings(std::string s) auto basePath = state->rootPath("."); // Try parsing as bindings - std::exception_ptr bindingsError; try { return state->parseReplBindings(s, basePath, staticEnv); } catch (ParseError &) { - bindingsError = std::current_exception(); } // Try with semicolon appended (for `inherit foo` shorthand) // Use original source (s) for error messages, not s + ";" try { return state->parseReplBindings(s + ";", s, basePath, staticEnv); - } catch (ParseError &) { - // Semicolon retry failed; rethrow the original bindings error - std::rethrow_exception(bindingsError); + } catch (ParseError & e) { + if (isIncompleteInput(e)) + throw IncompleteReplExpr(e.msg()); + // Semicolon retry also failed; not valid binding syntax. + return nullptr; } } @@ -888,10 +921,10 @@ void NixRepl::evalString(std::string s, Value & v) state->forceValue(v, v.determinePos(noPos)); } -void NixRepl::runNix(const std::string & program, OsStrings args, const std::optional & input) +void NixRepl::runNix(const std::string & program, OsStrings args) { if (runNixPtr) - (*runNixPtr)(program, std::move(args), input); + (*runNixPtr)(program, std::move(args)); else throw Error( "Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", @@ -922,7 +955,7 @@ ReplExitStatus AbstractNixRepl::runSimple(ref evalState, const ValMap // add 'extra' vars. for (auto & [name, value] : extraEnv) - repl->addVarToScope(repl->state->symbols.create(name), *value); + repl->addVarToScope(repl->state->symbols.create(name), **value); return repl->mainLoop(); } diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 8cc2bd713d07..84a72b597c5a 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -9,6 +9,8 @@ #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" +#include + #include #include #include @@ -19,6 +21,8 @@ namespace nix::unix { +void AbortServeSocket::anchor() {} + PeerInfo getPeerInfo(Descriptor remote) { PeerInfo peer; @@ -64,9 +68,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); @@ -83,6 +94,9 @@ PeerInfo getPeerInfo(Descriptor remote) for (auto & i : listeningSockets) fds.push_back({.fd = i.get(), .events = POLLIN}); + if (options.auxiliaryFd != INVALID_DESCRIPTOR) + fds.push_back({.fd = options.auxiliaryFd, .events = POLLIN}); + // Loop accepting connections. while (1) { try { @@ -95,7 +109,11 @@ PeerInfo getPeerInfo(Descriptor remote) throw SysError("polling for incoming connections"); } - for (auto & fd : fds) { + if (options.auxiliaryFd != INVALID_DESCRIPTOR && options.onAuxiliaryFdPollin && fds.back().revents & POLLIN) + /* Useful for reaping children. */ + options.onAuxiliaryFdPollin(); + + for (auto & fd : std::views::take(fds, listeningSockets.size())) { if (!fd.revents) continue; @@ -104,7 +122,6 @@ PeerInfo getPeerInfo(Descriptor remote) socklen_t remoteAddrLen = sizeof(remoteAddr); AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); if (!remote) { if (errno == EINTR) continue; @@ -114,6 +131,9 @@ PeerInfo getPeerInfo(Descriptor remote) handler(std::move(remote), [&]() { listeningSockets.clear(); }); } + } catch (AbortServeSocket &) { + /* Explicitly aborted, bail out. */ + throw; } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index c47704ce4112..90435b4095bb 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -47,16 +47,32 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixexprc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixexprc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixexprc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libexpr-c/meson.options b/src/libexpr-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libexpr-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 97680ac6bfe7..d78c504bd8c7 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) @@ -71,6 +50,7 @@ nix_err nix_expr_eval_from_string( nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); state->state.eval(parsedExpr, *value->value); state->state.forceValue(*value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -82,6 +62,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n try { state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos); state->state.forceValue(*value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -100,6 +81,7 @@ nix_err nix_value_call_multi( try { state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos); state->state.forceValue(*value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -110,6 +92,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * context->last_err_code = NIX_OK; try { state->state.forceValue(*value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -120,6 +103,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val context->last_err_code = NIX_OK; try { state->state.forceValueDeep(*value->value); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -129,23 +113,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -154,7 +135,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b498..3f7e4bf1df12 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -18,8 +18,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index a874d9a0861e..cf2fa2117273 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -1,7 +1,4 @@ -#include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/expr/value.hh" #include "nix_api_expr.h" @@ -44,6 +41,8 @@ nix_err nix_external_add_string_context(nix_c_context * context, nix_string_cont } // extern "C" +namespace { + class NixCExternalValue : public nix::ExternalValueBase { NixCExternalValueDesc & desc; @@ -169,6 +168,8 @@ class NixCExternalValue : public nix::ExternalValueBase virtual ~NixCExternalValue() override {}; }; +} // namespace + extern "C" { ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 589ebf9a8ec2..b0f96e3504c6 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,8 +1,6 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/store/path.hh" #include "nix/expr/primops.hh" #include "nix/expr/value.hh" @@ -13,7 +11,6 @@ #include "nix_api_util_internal.h" #include "nix_api_store_internal.h" #include "nix_api_value.h" -#include "nix/expr/value/context.hh" // Internal helper functions to check [in] and [out] `Value *` parameters static const nix::Value & check_value_not_null(const nix_value * value) @@ -395,6 +392,7 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value auto attr = v.attrs()->get(s); if (attr) { state->state.forceValue(*attr->value, nix::noPos); + state->state.waitForAllPaths(); return new_nix_value(attr->value, state->state.mem); } nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 0220bf68e2df..a01bfb280599 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -101,6 +101,8 @@ typedef enum { * @see Externals */ NIX_TYPE_EXTERNAL, + /** @brief Failed value. Contains an exception that can be rethrown. + */ NIX_TYPE_FAILED, } ValueType; diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 694fbc1fe789..937d7d3911c4 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -15,7 +16,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr-c"; + pname = "determinate-nix-expr-c"; inherit version; workDir = ./.; @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh index 376761d76325..4aa55688def2 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh @@ -12,21 +12,22 @@ class nix_api_expr_test : public nix_api_store_test { protected: - nix_api_expr_test() + void SetUp() override { + nix_api_store_test::SetUp(); nix_libexpr_init(ctx); state = nix_state_create(nullptr, nullptr, store); value = nix_alloc_value(nullptr, state); } - ~nix_api_expr_test() + void TearDown() override { nix_gc_decref(nullptr, value); nix_state_free(state); } - EvalState * state; - nix_value * value; + EvalState * state = nullptr; + nix_value * value = nullptr; }; } // namespace nixC diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index 68a0b8dea7d7..ed0eba94c30d 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -4,32 +4,26 @@ #include #include "nix/expr/value/context.hh" +#include "nix/store/tests/derived-path.hh" // IWYU pragma: keep namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index df28661b7e78..7d6f871d7acd 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -44,13 +44,12 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', sources, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 # is available. See also ../libutil/build.meson link_args : linker_export_flags + [ '-lrapidcheck' ], - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, ) diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 5cb4adaa8c46..1879a5716082 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-test-support"; + pname = "determinate-nix-util-test-support"; inherit version; workDir = ./.; diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index d6036601a948..f82bd1c552e8 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,7 +1,6 @@ -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include -#include "nix/store/tests/path.hh" #include "nix/expr/tests/value/context.hh" namespace rc { @@ -16,6 +15,15 @@ Gen Arbitrary::arb }); } +Gen Arbitrary::arbitrary() +{ + return gen::map(gen::arbitrary(), [](StorePath storePath) { + return NixStringContextElem::Path{ + .storePath = storePath, + }; + }); +} + Gen Arbitrary::arbitrary() { return gen::mapcat( @@ -31,6 +39,8 @@ Gen Arbitrary::arbitrary() case 2: return gen::map( gen::arbitrary(), [](NixStringContextElem a) { return a; }); + case 3: + return gen::map(gen::arbitrary(), [](NixStringContextElem a) { return a; }); default: assert(false); } diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index e9f9fcd0720d..3997280ee79e 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,6 +1,6 @@ #include #include -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" @@ -18,8 +18,6 @@ class DerivedPathExpressionTest : public LibExprTest // See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args TEST_F(DerivedPathExpressionTest, force_init) {} -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(DerivedPathExpressionTest, prop_opaque_path_round_trip, (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); @@ -61,6 +59,4 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(SingleDerivedPath{b} == d); } -#endif - } /* namespace nix */ diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc index 1b1c199bdff7..ea0736564f59 100644 --- a/src/libexpr-tests/dynamic-attrs-bench.cc +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static std::string mkDynamicAttrsExpr(size_t attrCount) { @@ -54,3 +54,5 @@ static void BM_EvalDynamicAttrs(benchmark::State & state) } BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000); + +} // namespace nix diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc48499a..9f2d1f92fa3a 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 7562a9da21ad..985564f13e7d 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/tests/libexpr.hh" -#include "nix/util/memory-source-accessor.hh" namespace nix { diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc index a5cd59154f22..9241e480480b 100644 --- a/src/libexpr-tests/get-drvs-bench.cc +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -6,8 +6,7 @@ #include "nix/store/store-open.hh" #include "nix/util/fmt.hh" -using namespace nix; - +namespace nix { namespace { struct GetDerivationsEnv @@ -64,3 +63,5 @@ static void BM_GetDerivationsAttrScan(benchmark::State & state) } BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000); + +} // namespace nix diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 000000000000..4c36424ecb55 --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 88a9d6684d5b..5a717d6bb053 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -1,18 +1,16 @@ #include #include "nix/store/tests/test-main.hh" -#include "nix/util/config-global.hh" - -using namespace nix; +#include "nix/util/configuration.hh" int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; // For pipe operator tests in trivial.cc - experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + nix::experimentalFeatureSettings.set("experimental-features", "pipe-operators"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index c5b72851da53..50d158209ba7 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -51,6 +51,7 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', 'nix_api_expr.cc', 'nix_api_external.cc', @@ -84,6 +85,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c3a3f2dd53b1..d9b203128553 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); @@ -577,6 +577,7 @@ struct DeploymentResourceState bool vm_created = false; }; +#if 0 static void primop_load_resource_input( void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { @@ -664,5 +665,6 @@ TEST_F(nix_api_expr_test, nix_expr_thunk_re_evaluation_after_deployment) assert_ctx_ok(); ASSERT_STREQ("vm-12345", result.c_str()); } +#endif } // namespace nixC diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index 885b9c1d2dd3..e17a52c31dfc 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" #include "nix_api_external.h" diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 830637f3ec50..01d15744a750 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -1,4 +1,3 @@ -#include "nix_api_store.h" #include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index ac5e893bf7d3..03014903fe0b 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -19,12 +19,12 @@ class CaptureLogger : public Logger return oss.str(); } - void log(Verbosity lvl, std::string_view s) override + void log(Verbosity lvl, std::string_view s) noexcept override { oss << s << std::endl; } - void logEI(const ErrorInfo & ei) override + void logEI(const ErrorInfo & ei) noexcept override { showErrorInfo(oss, ei, loggerSettings.showTrace.get()); } @@ -32,17 +32,17 @@ class CaptureLogger : public Logger class CaptureLogging { - std::unique_ptr oldLogger; + Logger * oldLogger; public: CaptureLogging() { - oldLogger = std::move(logger); - logger = std::make_unique(); + oldLogger = logger; + logger = new CaptureLogger(); } ~CaptureLogging() { - logger = std::move(oldLogger); + logger = oldLogger; } }; @@ -144,7 +144,7 @@ TEST_F(PrimOpTest, trace) CaptureLogging l; auto v = eval("builtins.trace \"test string 123\" 123"); ASSERT_THAT(v, IsIntEq(123)); - auto text = (dynamic_cast(logger.get()))->get(); + auto text = (dynamic_cast(logger))->get(); ASSERT_NE(text.find("test string 123"), std::string::npos); } @@ -637,7 +637,7 @@ TEST_F(PrimOpTest, toStringLambdaThrows) } class ToStringPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ToStringPrimOpTest, toString) @@ -651,7 +651,7 @@ TEST_P(ToStringPrimOpTest, toString) INSTANTIATE_TEST_SUITE_P( toString, ToStringPrimOpTest, - testing::Values( + ::testing::Values( CASE(R"("foo")", "foo"), CASE(R"(1)", "1"), CASE(R"([1 2 3])", "1 2 3"), @@ -799,7 +799,7 @@ TEST_F(PrimOpTest, splitVersion) } class CompareVersionsPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(CompareVersionsPrimOpTest, compareVersions) @@ -813,7 +813,7 @@ TEST_P(CompareVersionsPrimOpTest, compareVersions) INSTANTIATE_TEST_SUITE_P( compareVersions, CompareVersionsPrimOpTest, - testing::Values( + ::testing::Values( // The first two are weird cases. Intuition tells they should // be the same but they aren't. CASE(1.0, 1.0.0, -1), @@ -835,7 +835,7 @@ INSTANTIATE_TEST_SUITE_P( class ParseDrvNamePrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ParseDrvNamePrimOpTest, parseDrvName) @@ -857,7 +857,7 @@ TEST_P(ParseDrvNamePrimOpTest, parseDrvName) INSTANTIATE_TEST_SUITE_P( parseDrvName, ParseDrvNamePrimOpTest, - testing::Values( + ::testing::Values( std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"), std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git"))); diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc index 2eb17b212ab0..c0c61e58e993 100644 --- a/src/libexpr-tests/regex-cache-bench.cc +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) { @@ -44,3 +44,5 @@ static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) } BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex); + +} // namespace nix diff --git a/src/libexpr-tests/value/context.cc b/src/libexpr-tests/value/context.cc index fe3072b64ffd..c2c5cd3e8bbc 100644 --- a/src/libexpr-tests/value/context.cc +++ b/src/libexpr-tests/value/context.cc @@ -2,9 +2,8 @@ #include #include -#include "nix/store/tests/path.hh" -#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/value/context.hh" +#include "nix/store/store-dir-config.hh" namespace nix { @@ -121,8 +120,6 @@ TEST(NixStringContextElemTest, built_built_xp) NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -#ifndef COVERAGE - RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { ExperimentalFeatureSettings xpSettings; @@ -130,6 +127,4 @@ RC_GTEST_PROP(NixStringContextElemTest, prop_round_rip, (const NixStringContextE RC_ASSERT(o == NixStringContextElem::parse(o.to_string(), xpSettings)); } -#endif - } // namespace nix diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index 654a50b0ae0b..c9cabf3fa421 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -194,14 +194,14 @@ TEST_F(ValuePrintingTests, vFailed) try { throw Error("nope"); } catch (...) { - v.mkFailed(std::current_exception(), nullptr); + v.mkFailed(); } // Historically, a tried and then ignored value (e.g. through tryEval) was // reverted to the original thunk. - test(v, "«thunk»"); - test(v, ANSI_MAGENTA "«thunk»" ANSI_NORMAL, PrintOptions{.ansiColors = true}); + test(v, "«failed»"); + test(v, ANSI_MAGENTA "«failed»" ANSI_NORMAL, PrintOptions{.ansiColors = true}); } TEST_F(ValuePrintingTests, depthAttrs) diff --git a/src/libexpr-tests/value/value.cc b/src/libexpr-tests/value/value.cc index 285b586c0dfc..e22e70f6e336 100644 --- a/src/libexpr-tests/value/value.cc +++ b/src/libexpr-tests/value/value.cc @@ -13,7 +13,6 @@ TEST_F(ValueTest, unsetValue) { Value unsetValue; ASSERT_EQ(false, unsetValue.isValid()); - ASSERT_EQ(nThunk, unsetValue.type()); } TEST_F(ValueTest, vInt) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index c57b90112886..7ba510e43f7e 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -5,6 +5,10 @@ namespace nix { +void AttrPathNotFound::anchor() {} + +void NoPositionInfo::anchor() {} + static Strings parseAttrPath(std::string_view s) { Strings res; @@ -40,6 +44,14 @@ AttrPath AttrPath::parse(EvalState & state, std::string_view s) return res; } +AttrPath AttrPath::fromStrings(EvalState & state, const std::vector & attrNames) +{ + AttrPath res; + for (auto & attrName : attrNames) + res.push_back(state.symbols.create(attrName)); + return res; +} + std::string AttrPath::to_string(EvalState & state) const { return dropEmptyInitThenConcatStringsSep(".", state.symbols.resolve({*this})); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a97cef550e8d..d2f041a06bf8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -10,6 +10,8 @@ namespace nix::eval_cache { +void CachedEvalError::anchor() {} + CachedEvalError::CachedEvalError(ref cursor, Symbol attr) : CloneableError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) , cursor(cursor) @@ -106,7 +108,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; @@ -124,13 +126,23 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::FullAttrs) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::FullAttrs) + .apply(0, false) + .exec(); AttrId rowId = state->db.getLastInsertedRowId(); assert(rowId); for (auto & attr : attrs) - state->insertAttribute.use()(rowId)(symbols[attr])(AttrType::Placeholder) (0, false).exec(); + state->insertAttribute.use() + .apply(rowId) + .apply(symbols[attr]) + .apply(AttrType::Placeholder) + .apply(0, false) + .exec(); return rowId; }); @@ -150,10 +162,20 @@ struct AttrDb ctx.append(elem->view()); first = false; } - state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx) + state->insertAttributeWithContext.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::String) + .apply(s) + .apply(ctx) .exec(); } else { - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::String) (s).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::String) + .apply(s) + .exec(); } return state->db.getLastInsertedRowId(); @@ -165,7 +187,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Bool) (b ? 1 : 0).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Bool) + .apply(b ? 1 : 0) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -176,7 +203,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Int) (n).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Int) + .apply(n) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -187,9 +219,11 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute - .use()(key.first)(symbols[key.second])( - AttrType::ListOfStrings) (dropEmptyInitThenConcatStringsSep("\t", l)) + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::ListOfStrings) + .apply(dropEmptyInitThenConcatStringsSep("\t", l)) .exec(); return state->db.getLastInsertedRowId(); @@ -201,7 +235,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Placeholder) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Placeholder) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -212,7 +251,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Missing) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Missing) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -223,7 +267,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Misc) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Misc) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -234,7 +283,12 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use()(key.first)(symbols[key.second])(AttrType::Failed) (0, false).exec(); + state->insertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::Failed) + .apply(0, false) + .exec(); return state->db.getLastInsertedRowId(); }); @@ -244,7 +298,7 @@ struct AttrDb { auto state(_state->lock()); - auto queryAttribute(state->queryAttribute.use()(key.first)(symbols[key.second])); + auto queryAttribute(state->queryAttribute.use().apply(key.first).apply(symbols[key.second])); if (!queryAttribute.next()) return {}; @@ -257,7 +311,7 @@ struct AttrDb case AttrType::FullAttrs: { // FIXME: expensive, should separate this out. std::vector attrs; - auto queryAttributes(state->queryAttributes.use()(rowId)); + auto queryAttributes(state->queryAttributes.use().apply(rowId)); while (queryAttributes.next()) attrs.emplace_back(symbols.create(queryAttributes.getStr(0))); return {{rowId, attrs}}; @@ -307,11 +361,12 @@ EvalCache::EvalCache( Value * EvalCache::getRootValue() { - if (!value) { + auto value(this->value.lock()); + if (!*value) { debug("getting root value"); - value = allocRootValue(rootLoader()); + *value = RootValue(rootLoader()); } - return *value; + return **value; } ref EvalCache::getRoot() @@ -326,7 +381,7 @@ AttrCursor::AttrCursor( , cachedValue(std::move(cachedValue)) { if (value) - _value = allocRootValue(value); + *_value.lock() = RootValue(value); } AttrKey AttrCursor::getKey() @@ -342,18 +397,23 @@ AttrKey AttrCursor::getKey() Value & AttrCursor::getValue() { - if (!_value) { + /* Note: this lock is held while the value is being evaluated, + so concurrent calls block until the value is available. Lock + ordering is strictly child -> parent, so this cannot + deadlock. */ + auto value(_value.lock()); + if (!*value) { if (parent) { auto & vParent = parent->first->getValue(); root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); auto attr = vParent.attrs()->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); - _value = allocRootValue(attr->value); + *value = RootValue(attr->value); } else - _value = allocRootValue(root->getRootValue()); + *value = RootValue(root->getRootValue()); } - return **_value; + return ***value; } void AttrCursor::fetchCachedValue() @@ -364,23 +424,33 @@ void AttrCursor::fetchCachedValue() throw CachedEvalError(parent->first, parent->second); } -AttrPath AttrCursor::getAttrPath() const +AttrPath AttrCursor::getAttrPathRaw() const { if (parent) { - auto attrPath = parent->first->getAttrPath(); + auto attrPath = parent->first->getAttrPathRaw(); attrPath.push_back(parent->second); return attrPath; } else return {}; } -AttrPath AttrCursor::getAttrPath(Symbol name) const +AttrPath AttrCursor::getAttrPath() const +{ + return root->cleanupAttrPath(getAttrPathRaw()); +} + +AttrPath AttrCursor::getAttrPathRaw(Symbol name) const { - auto attrPath = getAttrPath(); + auto attrPath = getAttrPathRaw(); attrPath.push_back(name); return attrPath; } +AttrPath AttrCursor::getAttrPath(Symbol name) const +{ + return root->cleanupAttrPath(getAttrPathRaw(name)); +} + std::string AttrCursor::getAttrPathStr() const { return getAttrPath().to_string(root->state); @@ -554,18 +624,22 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - const StorePath & path = std::visit( + const StorePath * path = std::visit( overloaded{ - [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { return d.drvPath; }, - [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath->getBaseStorePath(); + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath * { return &d.drvPath; }, + [&](const NixStringContextElem::Built & b) -> const StorePath * { + return &b.drvPath->getBaseStorePath(); }, - [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath * { return &o.path; }, + [&](const NixStringContextElem::Path & p) -> const StorePath * { return nullptr; }, }, c.raw); - if (!root->state.store->isValidPath(path)) { - valid = false; - break; + if (path) { + root->state.store->addTempRoot(*path); + if (!root->state.store->isValidPath(*path)) { + valid = false; + break; + } } } if (valid) { @@ -707,13 +781,17 @@ StorePath AttrCursor::forceDerivation() auto aDrvPath = getAttr(root->state.s.drvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); drvPath.requireDerivation(); - if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { - /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ - aDrvPath->forceValue(); - if (!root->state.store->isValidPath(drvPath)) - throw Error( - "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + if (!settings.readOnlyMode) { + root->state.store->addTempRoot(drvPath); + if (!root->state.store->isValidPath(drvPath)) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + root->state.waitForPath(drvPath); + if (!root->state.store->isValidPath(drvPath)) + throw Error( + "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + } } return drvPath; } diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 38a60883bca7..ba72b05e98a6 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -6,7 +6,7 @@ namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } @@ -97,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template @@ -123,4 +123,32 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +void EvalBaseError::anchor() {} + +void ParseError::anchor() {} + +void EvalError::anchor() {} + +void AssertionError::anchor() {} + +void ThrownError::anchor() {} + +void Abort::anchor() {} + +void TypeError::anchor() {} + +void UndefinedVarError::anchor() {} + +void MissingArgumentError::anchor() {} + +void InfiniteRecursionError::anchor() {} + +void StackOverflowError::anchor() {} + +void InvalidPathError::anchor() {} + +void IFDError::anchor() {} + +void RecoverableEvalError::anchor() {} + } // namespace nix diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 0d25f38f64de..1aef8b844d4f 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,8 +1,6 @@ -#include "nix/util/error.hh" #include "nix/util/environment-variables.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" #include "nix/expr/value.hh" @@ -46,6 +44,88 @@ static void * oomHandler(size_t requested) throw std::bad_alloc(); } +static size_t getFreeMem() +{ + /* On Linux, use the `MemAvailable` or `MemFree` fields from + /proc/cpuinfo. */ +# ifdef __linux__ + { + std::unordered_map fields; + for (auto & line : + tokenizeString>(readFile(std::filesystem::path("/proc/meminfo")), "\n")) { + auto colon = line.find(':'); + if (colon == line.npos) + continue; + fields.emplace(line.substr(0, colon), trim(line.substr(colon + 1))); + } + + auto i = fields.find("MemAvailable"); + if (i == fields.end()) + i = fields.find("MemFree"); + if (i != fields.end()) { + auto kb = tokenizeString>(i->second, " "); + if (kb.size() == 2 && kb[1] == "kB") + return string2Int(kb[0]).value_or(0) * 1024; + } + } +# endif + + /* On non-Linux systems, conservatively assume that 25% of memory is free. */ + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize > 0 && pages > 0) + return (static_cast(pageSize) * static_cast(pages)) / 4; + return 0; +} + +/** + * When a thread goes into a coroutine, we lose its original sp until + * control flow returns to the thread. This causes Boehm GC to crash + * since it will scan memory between the coroutine's sp and the + * original stack base of the thread. Therefore, we detect when the + * current sp is outside of the original thread stack and push the + * entire thread stack instead, as an approximation. + * + * This is not optimal, because it causes the stack below sp to be + * scanned. However, we usually we don't have active coroutines during + * evaluation, so this is acceptable. + * + * Note that we don't scan coroutine stacks. It's currently assumed + * that we don't have GC roots in coroutines. + */ +void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) +{ + void *& sp = *sp_ptr; + auto pthread_id = reinterpret_cast(_pthread_id); + size_t osStackSize; + char * osStackHi; + char * osStackLo; + +# ifdef __APPLE__ + osStackSize = pthread_get_stacksize_np(pthread_id); + osStackHi = (char *) pthread_get_stackaddr_np(pthread_id); + osStackLo = osStackHi - osStackSize; +# else + pthread_attr_t pattr; + if (pthread_attr_init(&pattr)) + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); +# ifdef HAVE_PTHREAD_GETATTR_NP + if (pthread_getattr_np(pthread_id, &pattr)) + throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); +# else +# error "Need `pthread_attr_get_np`" +# endif + if (pthread_attr_getstack(&pattr, (void **) &osStackLo, &osStackSize)) + throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); + if (pthread_attr_destroy(&pattr)) + throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); + osStackHi = osStackLo + osStackSize; +# endif + + if (sp >= osStackHi || sp < osStackLo) // sp is outside the os stack + sp = osStackLo; +} + static inline void initGCReal() { /* Initialise the Boehm garbage collector. */ @@ -76,8 +156,26 @@ static inline void initGCReal() GC_set_oom_fn(oomHandler); - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases + GC_set_sp_corrector(&fixupBoehmStackPointer); + assert(GC_get_sp_corrector()); + + /* Funnel boehm warnings into debug logs. */ + GC_set_warn_proc([](char * msg, GC_word word) noexcept { + std::array buffer{}; + auto res = snprintf(buffer.data(), buffer.size(), msg, word); + /* Ignore garbage. */ + if (res < 0) + return; + + try { + debug("%s", chomp(std::string_view(buffer.data(), std::min(res, buffer.size() - 1)))); + } catch (...) { + /* Swallow all errors. */ + } + }); + + /* Set the initial heap size to something fairly big (80% of + free RAM, up to a maximum of 4 GiB) so that in most cases we don't need to garbage collect at all. (Collection has a fairly significant overhead.) The heap size can be overridden through libgc's GC_INITIAL_HEAP_SIZE environment variable. We @@ -88,15 +186,10 @@ static inline void initGCReal() if (!getEnv("GC_INITIAL_HEAP_SIZE")) { size_t size = 32 * 1024 * 1024; # if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) - size = maxSize; + size_t maxSize = 4ULL * 1024 * 1024 * 1024; + auto free = getFreeMem(); + size = std::max(size, std::min((size_t) (free * 0.5), maxSize)); # endif - debug("setting initial heap size to %1% bytes", size); GC_expand_hp(size); } } diff --git a/src/libexpr/eval-profiler.cc b/src/libexpr/eval-profiler.cc index 12c2f239df6e..9c7448d59076 100644 --- a/src/libexpr/eval-profiler.cc +++ b/src/libexpr/eval-profiler.cc @@ -171,6 +171,7 @@ class SampleStack : public EvalProfiler EvalState & state; std::chrono::nanoseconds sampleInterval; AutoCloseFD profileFd; + // FIXME: this needs to become per-thread to support multi-threaded evaluation. FrameStack stack; std::map callCount; std::chrono::time_point lastStackSample = diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 5cf0ae04304e..630f7b67067f 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -7,6 +7,8 @@ namespace nix { +void EvalSettings::anchor() {} + void DeprecatedWarnSetting::assign(const bool & v) { value = v; @@ -70,7 +72,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); @@ -113,9 +115,19 @@ bool EvalSettings::isPseudoUrl(std::string_view s) std::string EvalSettings::resolvePseudoUrl(std::string_view url) { - if (hasPrefix(url, "channel:")) - return "https://channels.nixos.org/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else + if (hasPrefix(url, "channel:")) { + auto realUrl = "https://channels.nixos.org/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + static bool haveWarned = false; + warnOnce( + haveWarned, + "Channels are deprecated in favor of flakes in Determinate Nix. " + "Instead of '%s', use '%s'. " + "See https://zero-to-nix.com for a guide to Nix flakes. " + "For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34.", + url, + realUrl); + return realUrl; + } else return std::string(url); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1ac9236d6609..0023c52ca7fd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,7 @@ #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" #include "nix/util/exit.hh" +#include "nix/util/signals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" #include "nix/util/environment-variables.hh" @@ -26,6 +27,8 @@ #include "nix/fetchers/tarball.hh" #include "nix/fetchers/input-cache.hh" #include "nix/util/current-process.hh" +#include "nix/store/async-path-writer.hh" +#include "nix/expr/parallel-eval.hh" #include "parser-tab.hh" @@ -42,10 +45,16 @@ #include #include #include +#include #include #include #include +#include + +#ifndef _WIN32 // TODO use portable implementation +# include +#endif #include "nix/util/strings-inline.hh" @@ -102,11 +111,6 @@ const StringData & StringData::make(EvalMemory & mem, std::string_view s) return res; } -RootValue allocRootValue(Value * v) -{ - return std::allocate_shared(traceable_allocator(), v); -} - // Pretty print types for assertion errors std::ostream & operator<<(std::ostream & os, const ValueType t) { @@ -197,20 +201,36 @@ PosIdx Value::determinePos(const PosIdx pos) const return attrs()->pos; case tLambda: return lambda().fun->pos; +#if 0 + // FIXME: disabled because reading from an app is racy. case tApp: return app().left->determinePos(pos); +#endif default: return pos; } #pragma GCC diagnostic pop } -bool Value::isTrivial() const +template<> +bool ValueStorage::isTrivial() const { - return !isa() - && (!isa() - || (dynamic_cast(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty()) - || dynamic_cast(thunk().expr) || dynamic_cast(thunk().expr)); + auto p1_ = p1; // must acquire before reading p0, since thunks can change + auto p0_ = p0.load(std::memory_order_acquire); + + auto pd = static_cast(p0_ & discriminatorMask); + + if (pd == pdThunk || pd == pdPending || pd == pdAwaited) { + bool isApp = p1_ & discriminatorMask; + if (isApp) + return false; + auto expr = untagPointer(p1_); + return (dynamic_cast(expr) && ((ExprAttrs *) expr)->dynamicAttrs->empty()) + || dynamic_cast(expr) || dynamic_cast(expr); + } + + else + return true; } static Symbol getName(const AttrName & name, EvalState & state, Env & env) @@ -232,6 +252,8 @@ EvalMemory::EvalMemory() assertGCInitialized(); } +[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext EvalState::evalContext; + EvalState::EvalState( const LookupPath & lookupPathFromArguments, ref store, @@ -274,8 +296,12 @@ EvalState::EvalState( instance if we're evaluating a file from the physical /nix/store while using a chroot store, and also for lazy mounted fetchTree. */ - auto accessor = settings.pureEval ? storeFS.cast() - : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + auto accessor = settings.pureEval + ? storeFS.cast() + : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}, storeFS.cast()); + + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) @@ -294,15 +320,21 @@ EvalState::EvalState( CanonPath("derivation-internal.nix"), #include "primops/derivation.nix.gen.hh" )} + , importedDrvToDerivation{internalFS->addFile( + CanonPath("imported-drv-to-derivation.nix"), +#include "imported-drv-to-derivation.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) - , srcToStore(make_ref()) + , asyncPathWriter(AsyncPathWriter::make(store)) , importResolutionCache(make_ref()) , fileEvalCache(make_ref()) + , positionToDocComment(make_ref()) + , lookupPathResolved(make_ref()) , regexCache(makeRegexCache()) #if NIX_USE_BOEHMGC , baseEnvP(std::allocate_shared(traceable_allocator(), &mem.allocEnv(BASE_ENV_SIZE))) @@ -311,12 +343,26 @@ EvalState::EvalState( , baseEnv(mem.allocEnv(BASE_ENV_SIZE)) #endif , staticBaseEnv{std::make_shared(nullptr, nullptr)} -{ + , countCalls(getEnv("NIX_COUNT_CALLS").value_or("0") != "0") + , primOpCalls(make_ref()) + , functionCalls(make_ref()) + , attrSelects(make_ref()) + , executor{make_ref(settings)} +{ +#ifndef _WIN32 + static std::once_flag stackSizeBumped; + std::call_once(stackSizeBumped, []() { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a + // hard limit slightly under that, so we round it down a bit. + nix::ensureStackSizeAtLeast(60 * 1024 * 1024); + }); +#endif + corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); - countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Construct the Nix expression search path. */ @@ -457,7 +503,8 @@ void EvalState::checkURI(const std::string & uri0) Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); - *v2 = v; + // Do a raw copy since `operator =` barfs on thunks. + memcpy((char *) v2, (char *) &v, sizeof(Value)); addConstant(name, v2, info); return v2; } @@ -473,8 +520,10 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) We might know the type of a thunk in advance, so be allowed to just write it down in that case. */ - if (auto gotType = v->type(); gotType != nThunk) - assert(info.type == gotType); + if (v->isFinished()) { + if (auto gotType = v->type(); gotType != nThunk) + assert(info.type == gotType); + } /* Install value the base environment. */ staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); @@ -544,7 +593,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) v->mkPrimOp(new PrimOp(primOp)); if (primOp.internal) - internalPrimOps.emplace(primOp.name, v); + internalPrimOps.emplace(primOp.name, RootValue(v)); else { staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; @@ -647,27 +696,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { - if (!env.values[0]->isThunk()) { + if (env.values[0]->isFinished()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -688,7 +744,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; @@ -715,14 +771,14 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (se.isWith && !env.values[0]->isThunk()) { + if (se.isWith && env.values[0]->isFinished()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) - vm.insert_or_assign(std::string(st[j.name]), j.value); + vm.insert_or_assign(std::string(st[j.name]), RootValue(j.value)); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); + vm.insert_or_assign(std::string(st[i.first]), RootValue(env.values[i.second])); } } } @@ -911,7 +967,7 @@ void Value::mkPath(const SourcePath & path, EvalMemory & mem) forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); if (auto j = env->values[0]->attrs()->get(var.name)) { if (countCalls) [[unlikely]] - attrSelects[j->pos]++; + attrSelects->try_emplace_or_visit(j->pos, 1, [](auto & i) { i.second++; }); return j->value; } if (!fromWith->parentWith) [[unlikely]] @@ -954,7 +1010,14 @@ void EvalState::mkPos(Value & v, PosIdx p) auto origin = positions.originOf(p); if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); - attrs.alloc(s.file).mkString(path->path.abs(), mem); + if (path->accessor == rootFS && store->isInStore(path->path.abs())) + // FIXME: only do this for virtual store paths? + attrs.alloc(s.file).mkString( + path->path.abs(), + {NixStringContextElem::Path{.storePath = store->toStorePath(path->path.abs()).first}}, + mem); + else + attrs.alloc(s.file).mkString(path->path.abs(), mem); makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column)); v.mkAttrs(attrs); } else @@ -1002,6 +1065,7 @@ std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p) auto optStaticOutputPath = std::visit( overloaded{ [&](const SingleDerivedPath::Opaque & o) { + waitForPath(o.path); auto drv = store->readDerivation(o.path); auto i = drv.outputs.find(b.output); if (i == drv.outputs.end()) @@ -1072,15 +1136,16 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) return &v; } +namespace { + /** * A helper `Expr` class to lets us parse and evaluate Nix expressions * from a thunk, ensuring that every file is parsed/evaluated only * once (via the thunk stored in `EvalState::fileEvalCache`). */ -struct ExprParseFile : Expr, gc +struct ExprParseFile : Expr { - // FIXME: make this a reference (see below). - SourcePath path; + SourcePath & path; bool mustBeTrivial; ExprParseFile(SourcePath & path, bool mustBeTrivial) @@ -1115,6 +1180,8 @@ struct ExprParseFile : Expr, gc } }; +} // namespace + void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { auto resolvedPath = getConcurrent(*importResolutionCache, path); @@ -1124,28 +1191,28 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) importResolutionCache->emplace(path, *resolvedPath); } - if (auto v2 = getConcurrent(*fileEvalCache, *resolvedPath)) { - forceValue(**v2, noPos); - v = **v2; - return; + { + Value * v2 = nullptr; + fileEvalCache->cvisit(*resolvedPath, [&](auto & i) { v2 = *i.second; }); + if (v2) { + forceValue(*v2, noPos); + v = *v2; + return; + } } Value * vExpr; - // FIXME: put ExprParseFile on the stack instead of the heap once - // https://github.com/NixOS/nix/pull/13930 is merged. That will ensure - // the post-condition that `expr` is unreachable after - // `forceValue()` returns. - auto expr = new ExprParseFile{*resolvedPath, mustBeTrivial}; + ExprParseFile expr{*resolvedPath, mustBeTrivial}; fileEvalCache->try_emplace_and_cvisit( *resolvedPath, - nullptr, + RootValue(nullptr), [&](auto & i) { vExpr = allocValue(); - vExpr->mkThunk(&baseEnv, expr); - i.second = vExpr; + vExpr->mkThunk(&baseEnv, &expr); + *i.second = vExpr; }, - [&](auto & i) { vExpr = i.second; }); + [&](auto & i) { vExpr = *i.second; }); forceValue(*vExpr, noPos); @@ -1157,7 +1224,8 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); - positions.clear(); + lookupPathResolved->clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) @@ -1327,7 +1395,12 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) sort = true; } - bindings.bindings->pos = pos; + /* Empty attrsets share the static Bindings::emptyBindings, which we + must not write to: apart from being a data race, it causes false + sharing on emptyBindings' cache line (which may also hold other hot + globals such as Counter::enabled) between all evaluator threads. */ + if (bindings.bindings != &Bindings::emptyBindings) + bindings.bindings->pos = pos; v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted()); } @@ -1445,10 +1518,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) vAttrs = j->value; pos2 = j->pos; if (state.countCalls) - state.attrSelects[pos2]++; + state.attrSelects->try_emplace_or_visit(pos2, 1, [](auto & i) { i.second++; }); } - state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos)); + state.forceValue(*vAttrs, pos2 ? pos2 : this->pos); } catch (Error & e) { if (pos2) { @@ -1507,6 +1580,8 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } +[[gnu::tls_model("initial-exec")]] thread_local size_t EvalState::callDepth = 0; + void EvalState::callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos) { auto _level = addCallDepth(pos); @@ -1522,15 +1597,16 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, forceValue(fun, pos); - Value vCur(fun); + Value vCur = fun; auto makeAppChain = [&]() { - vRes = vCur; for (auto arg : args) { auto fun2 = allocValue(); - *fun2 = vRes; - vRes.mkPrimOpApp(fun2, arg); + *fun2 = vCur; + vCur.reset(); + vCur.mkPrimOpApp(fun2, arg); } + vRes = vCur; }; const Attr * functor; @@ -1626,6 +1702,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, lambda.name ? concatStrings("'", symbols[lambda.name], "'") : "anonymous lambda") : nullptr; + vCur.reset(); lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { @@ -1657,10 +1734,12 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, nrPrimOpCalls++; if (countCalls) - primOpCalls[fn->name]++; + primOpCalls->try_emplace_or_visit(fn->name, 1, [](auto & i) { i.second++; }); try { - fn->impl(*this, vCur.determinePos(noPos), args.data(), vCur); + auto pos = vCur.determinePos(noPos); + vCur.reset(); + fn->impl(*this, pos, args.data(), vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -1682,6 +1761,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, assert(primOp->isPrimOp()); auto arity = primOp->primOp()->arity; auto argsLeft = arity - argsDone; + assert(argsLeft); if (args.size() < argsLeft) { /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ @@ -1702,7 +1782,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, auto fn = primOp->primOp(); nrPrimOpCalls++; if (countCalls) - primOpCalls[fn->name]++; + primOpCalls->try_emplace_or_visit(fn->name, 1, [](auto & i) { i.second++; }); try { // TODO: @@ -1710,7 +1790,9 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 // etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - fn->impl(*this, vCur.determinePos(noPos), vArgs, vCur); + auto pos = vCur.determinePos(noPos); + vCur.reset(); + fn->impl(*this, pos, vArgs, vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -1727,6 +1809,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, heap-allocate a copy and use that instead. */ Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; + vCur.reset(); try { callFunction(*functor->value, args2, vCur, functor->pos); } catch (Error & e) { @@ -1773,7 +1856,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // prevents tail-call optimisation. void EvalState::incrFunctionCall(ExprLambda * fun) { - functionCalls[fun]++; + functionCalls->try_emplace_or_visit(fun, 1, [](auto & i) { i.second++; }); } void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res) @@ -1914,8 +1997,12 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator")); } -void ExprOpUpdate::eval(EvalState & state, Value & v, Value & v1, Value & v2) +void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) { + Value v1, v2; + state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator"); + state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator"); + state.nrOpUpdates++; const Bindings & bindings1 = *v1.attrs(); @@ -1989,42 +2076,6 @@ void ExprOpUpdate::eval(EvalState & state, Value & v, Value & v1, Value & v2) state.nrOpUpdateValuesCopied += v.attrs()->size(); } -void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) -{ - UpdateQueue q; - evalForUpdate(state, env, q); - - Value vTmp; - vTmp.mkAttrs(&Bindings::emptyBindings); - - for (auto & rhs : std::views::reverse(q)) { - /* Remember that queue is sorted rightmost attrset first. */ - eval(state, /*v=*/vTmp, /*v1=*/vTmp, /*v2=*/rhs); - } - - v = vTmp; -} - -void Expr::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) -{ - Value v; - state.evalAttrs(env, this, v, getPos(), errorCtx); - q.push_back(v); -} - -void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q) -{ - /* Output rightmost attrset first to the merge queue as the one - with the most priority. */ - e2->evalForUpdate(state, env, q, "in the right operand of the update (//) operator"); - e1->evalForUpdate(state, env, q, "in the left operand of the update (//) operator"); -} - -void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) -{ - evalForUpdate(state, env, q); -} - void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) { Value v1; @@ -2032,22 +2083,21 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) Value v2; e2->eval(state, env, v2); Value * lists[2] = {&v1, &v2}; - state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate"); + state.concatLists(v, lists, pos, "while evaluating one of the elements to concatenate"); } -void EvalState::concatLists( - Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx) +void EvalState::concatLists(Value & v, std::span lists, const PosIdx pos, std::string_view errorCtx) { nrListConcats++; - Value * nonEmpty = 0; + Value * nonEmpty = nullptr; size_t len = 0; - for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos, errorCtx); - auto l = lists[n]->listSize(); + for (auto * list : lists) { + forceList(*list, pos, errorCtx); + auto l = list->listSize(); len += l; if (l) - nonEmpty = lists[n]; + nonEmpty = list; } if (nonEmpty && len == nonEmpty->listSize()) { @@ -2057,12 +2107,13 @@ void EvalState::concatLists( auto list = buildList(len); auto out = list.elems; - for (size_t n = 0, pos = 0; n < nrLists; ++n) { - auto listView = lists[n]->listView(); - auto l = listView.size(); - if (l) - memcpy(out + pos, listView.data(), l * sizeof(Value *)); - pos += l; + size_t pos2 = 0; + for (auto * l : lists) { + auto listView = l->listView(); + auto n = listView.size(); + if (n) + memcpy(out + pos2, listView.data(), n * sizeof(Value *)); + pos2 += n; } v.mkList(list); } @@ -2144,7 +2195,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) } else if (firstType == nFloat) { v.mkFloat(nf); } else if (firstType == nPath) { - if (!context.empty()) + if (hasContext(context)) state.error("a string that refers to a store path cannot be appended to a path") .atPos(pos) .withFrame(env, *this) @@ -2172,67 +2223,6 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) state.mkPos(v, pos); } -void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v) -{ - throwInfiniteRecursionError(state, v); -} - -[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value & v) -{ - state.error("infinite recursion encountered").atPos(v.determinePos(noPos)).debugThrow(); -} - -// always force this to be separate, otherwise forceValue may inline it and take -// a massive perf hit -[[gnu::noinline]] -void EvalState::handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos) -{ - if (!env) - tryFixupBlackHolePos(v, pos); - - auto e = std::current_exception(); - Value * recovery = nullptr; - try { - std::rethrow_exception(e); - } catch (const RecoverableEvalError & e) { - recovery = allocValue(); - } catch (...) { - } - if (recovery) { - recovery->mkThunk(env, expr); - } - v.mkFailed(e, recovery); -} - -[[gnu::noinline]] -void EvalState::handleEvalExceptionForApp(Value & v, const Value & savedApp) -{ - auto e = std::current_exception(); - Value * recovery = nullptr; - try { - std::rethrow_exception(e); - } catch (const RecoverableEvalError & e) { - recovery = allocValue(); - } catch (...) { - } - if (recovery) { - *recovery = savedApp; - } - v.mkFailed(e, recovery); -} - -[[gnu::noinline]] -void EvalState::handleEvalFailed(Value & v, const PosIdx pos) -{ - assert(v.isFailed()); - if (auto recoveryValue = v.failed().recoveryValue) { - v = *recoveryValue; - forceValue(v, pos); - } else { - v.failed().rethrow(); - } -} - void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) { if (!v.isBlackhole()) @@ -2263,6 +2253,7 @@ void EvalState::forceValueDeep(Value & v) for (auto & i : *v.attrs()) try { // If the value is a thunk, we're evaling. Otherwise no trace necessary. + // FIXME: race, thunk might be updated by another thread auto dts = state.debugRepl && i.value->isThunk() ? makeDebugTraceStacker( state, *i.value->thunk().expr, @@ -2415,13 +2406,17 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s { auto s = forceString(v, pos, errorCtx); if (v.context()) { - auto ctxElem = NixStringContextElem::parse((*v.context()->begin())->view()); - error( - "the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string_view(), - ctxElem.display(*store)) - .withTrace(pos, errorCtx) - .debugThrow(); + NixStringContext context; + copyContext(v, context); + if (hasContext(context)) { + auto ctxElem = NixStringContextElem::parse((*v.context()->begin())->view()); + error( + "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string_view(), + ctxElem.display(*store)) + .withTrace(pos, errorCtx) + .debugThrow(); + } } return s; } @@ -2478,14 +2473,24 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { + // FIXME: instead of copying the path to the store, we could + // return a virtual store path that lazily copies the path to + // the store in devirtualize(). if (!canonicalizePath && !copyToStore) { // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. return v.pathStrView(); } else if (copyToStore) { - return store->printStorePath(copyPathToStore(context, v.path())); + return store->printStorePath(copyPathToStore(context, v.path(), v.determinePos(pos))); } else { - return std::string{v.path().path.abs()}; + auto path = v.path(); + if (path.accessor == rootFS && store->isInStore(path.path.abs())) { + try { + context.insert(NixStringContextElem::Path{.storePath = store->toStorePath(path.path.abs()).first}); + } catch (Error &) { + } + } + return std::string(path.path.abs()); } } @@ -2557,28 +2562,21 @@ BackedStringView EvalState::coerceToString( .debugThrow(); } -StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos) { if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto dstPathCached = getConcurrent(*srcToStore, path); - - auto dstPath = dstPathCached ? *dstPathCached : [&]() { - auto dstPath = fetchToStore( - fetchSettings, - *store, - path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, - path.baseName(), - ContentAddressMethod::Raw::NixArchive, - nullptr, - repair); - allowPath(dstPath); - srcToStore->try_emplace(path, dstPath); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); - return dstPath; - }(); + auto dstPath = fetchToStore( + fetchSettings, + *store, + path.resolveSymlinks(SymlinkResolution::Ancestors), + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, + computeBaseName(path, pos), + ContentAddressMethod::Raw::NixArchive, + nullptr, + repair); + allowPath(dstPath); context.insert(NixStringContextElem::Opaque{.path = dstPath}); return dstPath; @@ -2613,7 +2611,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return rootPath(path); + return rootPath(CanonPath(path)); } StorePath @@ -2622,7 +2620,9 @@ EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & con auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path) + .withTrace(pos, errorCtx) + .debugThrow(); } std::pair EvalState::coerceToSingleDerivedPathUnchecked( @@ -2646,6 +2646,9 @@ std::pair EvalState::coerceToSingleDerivedP .debugThrow(); }, [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); }, + [&](NixStringContextElem::Path && p) -> SingleDerivedPath { + error("string '%s' has no context", s).withTrace(pos, errorCtx).debugThrow(); + }, }, ((NixStringContextElem &&) *context.begin()).raw); return { @@ -3090,6 +3093,11 @@ void EvalState::printStatistics() topObj["nrOpUpdates"] = nrOpUpdates.load(); topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load(); topObj["nrThunks"] = nrThunks.load(); + topObj["nrThunksAwaited"] = nrThunksAwaited.load(); + topObj["nrThunksAwaitedSlow"] = nrThunksAwaitedSlow.load(); + topObj["nrSpuriousWakeups"] = nrSpuriousWakeups.load(); + topObj["maxWaiting"] = maxWaiting.load(); + topObj["waitingTime"] = microsecondsWaiting / (double) 1000000; topObj["nrAvoided"] = nrAvoided.load(); topObj["nrLookups"] = nrLookups.load(); topObj["nrPrimOpCalls"] = nrPrimOpCalls.load(); @@ -3103,11 +3111,16 @@ void EvalState::printStatistics() #endif if (countCalls) { - topObj["primops"] = primOpCalls; + { + auto & obj = topObj["primops"]; + obj = json::object(); + primOpCalls->visit_all([&](auto & i) { obj[i.first] = i.second; }); + } { auto & list = topObj["functions"]; list = json::array(); - for (auto & [fun, count] : functionCalls) { + functionCalls->visit_all([&](auto & i) { + auto & [fun, count] = i; json obj = json::object(); if (fun->name) obj["name"] = (std::string_view) symbols[fun->name]; @@ -3121,12 +3134,12 @@ void EvalState::printStatistics() } obj["count"] = count; list.push_back(obj); - } + }); } { - auto list = topObj["attributes"]; + auto & list = topObj["attributes"]; list = json::array(); - for (auto & i : attrSelects) { + attrSelects->visit_all([&](auto & i) { json obj = json::object(); if (auto pos = positions[i.first]) { if (auto path = std::get_if(&pos.origin)) @@ -3136,15 +3149,15 @@ void EvalState::printStatistics() } obj["count"] = i.second; list.push_back(obj); - } + }); } } if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + auto list = json::array(); + symbols.dump([&](std::string_view s) { list.emplace_back(std::string(s)); }); // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto & list = topObj["symbols"]; - symbols.dump([&](std::string_view s) { list.emplace_back(s); }); + topObj["symbols"] = std::move(list); } if (outPath == "-") { std::cerr << topObj.dump(2) << std::endl; @@ -3252,14 +3265,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3273,19 +3300,23 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; - auto i = lookupPathResolved.find(value); - if (i != lookupPathResolved.end()) - return i->second; - - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + if (auto cached = getConcurrent(*lookupPathResolved, value)) + return *cached; + + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); - lookupPathResolved.emplace(value, res); + } + lookupPathResolved->emplace(std::string(value), res); return res; }; @@ -3345,19 +3376,21 @@ Expr * EvalState::parse( const SourcePath & basePath, const std::shared_ptr & staticEnv) { - DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath - DocCommentMap * docComments = &tmpDocComments; + auto tmpDocComments = make_ref(); - if (auto sourcePath = std::get_if(&origin)) { - auto [it, _] = positionToDocComment.try_emplace(*sourcePath); - docComments = &it->second; - } - - auto result = - parseExprFromBuf(text, length, origin, basePath, mem.exprs, symbols, settings, positions, *docComments, rootFS); + auto result = parseExprFromBuf( + text, length, origin, basePath, mem.exprs, symbols, settings, positions, *tmpDocComments, rootFS); result->bindVars(*this, staticEnv); + if (auto sourcePath = std::get_if(&origin)) + /* A single file might appear multiple times in PosTable if it's + parsed by scopedImport. If we are the first then emplace into the map, otherwise + copy our positions into the existing map. */ + positionToDocComment->emplace_or_visit(*sourcePath, tmpDocComments, [&tmpDocComments](auto & kv) { + kv.second->insert(tmpDocComments->begin(), tmpDocComments->end()); + }); + return result; } @@ -3368,20 +3401,22 @@ ExprAttrs * EvalState::parseReplBindings( const SourcePath & basePath, const std::shared_ptr & staticEnv) { - DocCommentMap tmpDocComments; - DocCommentMap * docComments = &tmpDocComments; - - if (auto sourcePath = std::get_if(&origin)) { - auto [it, _] = positionToDocComment.try_emplace(*sourcePath); - docComments = &it->second; - } + auto tmpDocComments = make_ref(); auto bindings = parseReplBindingsFromBuf( - text, length, origin, basePath, mem.exprs, symbols, settings, positions, *docComments, rootFS); + text, length, origin, basePath, mem.exprs, symbols, settings, positions, *tmpDocComments, rootFS); assert(bindings); bindings->bindVars(*this, staticEnv); + if (auto sourcePath = std::get_if(&origin)) + /* A single file might appear multiple times in PosTable if it's + parsed by scopedImport. If we are the first then emplace into the map, otherwise + copy our positions into the existing map. */ + positionToDocComment->emplace_or_visit(*sourcePath, tmpDocComments, [&tmpDocComments](auto & kv) { + kv.second->insert(tmpDocComments->begin(), tmpDocComments->end()); + }); + return bindings; } @@ -3392,14 +3427,12 @@ DocComment EvalState::getDocCommentForPos(PosIdx pos) if (!path) return {}; - auto table = positionToDocComment.find(*path); - if (table == positionToDocComment.end()) - return {}; - - auto it = table->second.find(pos); - if (it == table->second.end()) - return {}; - return it->second; + DocComment result; + positionToDocComment->visit(*path, [&](const auto & kv) { + if (auto it = kv.second->find(pos); it != kv.second->end()) + result = it->second; + }); + return result; } std::string ExternalValueBase::coerceToString( @@ -3427,8 +3460,28 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } +void EvalState::waitForPath(const StorePath & path) +{ + asyncPathWriter->waitForPath(path); +} + +void EvalState::waitForPath(const SingleDerivedPath & path) +{ + std::visit( + overloaded{ + [&](const DerivedPathOpaque & p) { waitForPath(p.path); }, + [&](const SingleDerivedPathBuilt & p) { waitForPath(*p.drvPath); }, + }, + path.raw()); +} + +void EvalState::waitForAllPaths() +{ + asyncPathWriter->waitForAllPaths(); +} + } // namespace nix diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 03a1aa455ce0..1acc591b05b7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -5,7 +5,6 @@ #include "nix/store/path-with-outputs.hh" #include -#include namespace nix { @@ -163,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; diff --git a/src/libexpr/include/nix/expr/attr-path.hh b/src/libexpr/include/nix/expr/attr-path.hh index fd48705b8b7b..25384f5c4c87 100644 --- a/src/libexpr/include/nix/expr/attr-path.hh +++ b/src/libexpr/include/nix/expr/attr-path.hh @@ -25,6 +25,8 @@ struct AttrPath : std::vector static AttrPath parse(EvalState & state, std::string_view s); + static AttrPath fromStrings(EvalState & state, const std::vector & attrNames); + std::string to_string(EvalState & state) const; std::vector resolve(EvalState & state) const; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 8dfe052134a1..4a360970ba21 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -28,11 +28,7 @@ enum struct Diagnose { Fatal, }; -template<> -Diagnose BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(Diagnose) /** * Check a diagnostic setting and either do nothing, log a warning, or throw an error. @@ -48,7 +44,7 @@ std::string BaseSetting::to_string() const; * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -68,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index d2ead2bb4614..df237d704b23 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -8,6 +8,7 @@ #include #include +#include "nix/expr/root-value.hh" namespace nix::eval_cache { @@ -16,6 +17,9 @@ class AttrCursor; struct CachedEvalError : CloneableError { +private: + void anchor() override; +public: const ref cursor; const Symbol attr; @@ -35,10 +39,16 @@ class EvalCache : public std::enable_shared_from_this friend struct CachedEvalError; std::shared_ptr db; + +public: EvalState & state; + + std::function cleanupAttrPath = [](AttrPath && attrPath) { return std::move(attrPath); }; + +private: typedef fun RootLoader; RootLoader rootLoader; - RootValue value; + Sync value; Value * getRootValue(); @@ -99,10 +109,13 @@ class AttrCursor : public std::enable_shared_from_this friend class EvalCache; friend struct CachedEvalError; - ref root; +public: + const ref root; + +private: using Parent = std::optional, Symbol>>; - Parent parent; - RootValue _value; + const Parent parent; + Sync _value; std::optional> cachedValue; AttrKey getKey(); @@ -127,8 +140,12 @@ public: AttrPath getAttrPath() const; + AttrPath getAttrPathRaw() const; + AttrPath getAttrPath(Symbol name) const; + AttrPath getAttrPathRaw(Symbol name) const; + std::string getAttrPathStr() const; std::string getAttrPathStr(Symbol name) const; diff --git a/src/libexpr/include/nix/expr/eval-error.hh b/src/libexpr/include/nix/expr/eval-error.hh index 68aa7b0643a2..8dce750e073c 100644 --- a/src/libexpr/include/nix/expr/eval-error.hh +++ b/src/libexpr/include/nix/expr/eval-error.hh @@ -23,6 +23,9 @@ class EvalBaseError : public CloneableError { template friend class EvalErrorBuilder; + + void anchor() override; + public: EvalState & state; @@ -61,8 +64,11 @@ MakeError(InfiniteRecursionError, EvalError); * Inherits from EvalBaseError (not EvalError) because resource exhaustion * should not be cached. */ -struct StackOverflowError : public CloneableError +class StackOverflowError : public CloneableError { + void anchor() override; + +public: StackOverflowError(EvalState & state) : CloneableError(state, "stack overflow; max-call-depth exceeded") { @@ -79,8 +85,10 @@ MakeError(IFDError, EvalBaseError); */ MakeError(RecoverableEvalError, EvalBaseError); -struct InvalidPathError : public CloneableError +class InvalidPathError : public CloneableError { + void anchor() override; + public: StorePath path; diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index 9163238d4f6e..b724762a80c9 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -32,7 +32,7 @@ Value * EvalMemory::allocValue() #if NIX_USE_BOEHMGC /* Allocation cache for GC'd Value objects. Boehm GC is already a global resource, so thread_local is a natural solution. Multiple EvalState instances on the same thread will reuse the same cache. */ - static thread_local std::shared_ptr valueAllocCache{ + [[gnu::tls_model("initial-exec")]] static thread_local std::shared_ptr valueAllocCache{ std::allocate_shared(traceable_allocator(), nullptr)}; /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). @@ -70,7 +70,7 @@ Env & EvalMemory::allocEnv(size_t size) if (size == 1) { /* Allocation cache for size-1 Env objects. Boehm GC is already a global resource, so thread_local is a natural solution. Multiple EvalState instances on the same thread will reuse the same cache. */ - static thread_local std::shared_ptr env1AllocCache{ + [[gnu::tls_model("initial-exec")]] static thread_local std::shared_ptr env1AllocCache{ std::allocate_shared(traceable_allocator(), nullptr)}; /* see allocValue for explanations. */ if (!*env1AllocCache) { @@ -92,34 +92,68 @@ Env & EvalMemory::allocEnv(size_t size) return *env; } -[[gnu::always_inline]] -void EvalState::forceValue(Value & v, const PosIdx pos) +/** + * An identifier of the current thread for deadlock detection, stored + * in p0 of pending/awaited thunks. We're not using std::thread::id + * because it's not guaranteed to fit. + */ +[[gnu::tls_model("initial-exec")]] extern thread_local uint32_t myEvalThreadId; + +template +void ValueStorage>>::force( + EvalState & state, PosIdx pos) { - if (v.isThunk()) { - Env * env = v.thunk().env; - assert(env || v.isBlackhole()); - Expr * expr = v.thunk().expr; - try { - v.mkBlackhole(); - if (env) [[likely]] - expr->eval(*this, *env, v); - else - ExprBlackHole::throwInfiniteRecursionError(*this, v); - } catch (...) { - handleEvalExceptionForThunk(env, expr, v, pos); - throw; - } - } else if (v.isApp()) { - Value savedApp = v; + auto p0_ = p0.load(std::memory_order_acquire); + + auto pd = static_cast(p0_ & discriminatorMask); + + if (pd == pdThunk) { try { - callFunction(*v.app().left, *v.app().right, v, pos); + // The value we get here is only valid if we can set the + // thunk to pending. + auto p1_ = p1; + + // Atomically set the thunk to "pending". + if (!p0.compare_exchange_strong( + p0_, + pdPending | (myEvalThreadId << discriminatorBits), + std::memory_order_acquire, + std::memory_order_acquire)) { + pd = static_cast(p0_ & discriminatorMask); + if (pd == pdPending || pd == pdAwaited) { + // The thunk is already "pending" or "awaited", so + // we need to wait for it. + p0_ = waitOnThunk(state, p0_); + goto done; + } + assert(pd != pdThunk); + // Another thread finished this thunk, no need to wait. + goto done; + } + + bool isApp = p1_ & discriminatorMask; + if (isApp) { + auto left = untagPointer(p0_); + auto right = untagPointer(p1_); + state.callFunction(*left, *right, (Value &) *this, pos); + } else { + auto env = untagPointer(p0_); + auto expr = untagPointer(p1_); + expr->eval(state, *env, (Value &) *this); + } } catch (...) { - handleEvalExceptionForApp(v, savedApp); + state.tryFixupBlackHolePos((Value &) *this, pos); + setStorage(new Value::Failed{.ex = std::current_exception()}); throw; } - } else if (v.isFailed()) { - handleEvalFailed(v, pos); } + + else if (pd == pdPending || pd == pdAwaited) + p0_ = waitOnThunk(state, p0_); + +done: + if (InternalType(p0_ & 0xff) == tFailed) + std::rethrow_exception((std::bit_cast(p1))->ex); } [[gnu::always_inline]] diff --git a/src/libexpr/include/nix/expr/eval-profiler-settings.hh b/src/libexpr/include/nix/expr/eval-profiler-settings.hh index 32138e7f13f0..4fec151b8061 100644 --- a/src/libexpr/include/nix/expr/eval-profiler-settings.hh +++ b/src/libexpr/include/nix/expr/eval-profiler-settings.hh @@ -7,10 +7,6 @@ namespace nix { enum struct EvalProfilerMode { disabled, flamegraph }; -template<> -EvalProfilerMode BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(EvalProfilerMode) } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d9dba95370b1..76fe402e11c6 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -43,6 +43,10 @@ public: struct EvalSettings : Config { +private: + void anchor() override; + +public: /** * Function used to interpret look path entries of a given scheme. * @@ -71,7 +75,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); @@ -122,7 +133,7 @@ struct EvalSettings : Config - `$HOME/.nix-defexpr/channels` - The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user. + The user channel link pointing to the current state of channels for the current user. - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` @@ -132,7 +143,7 @@ struct EvalSettings : Config The current state of all channels for the `root` user. - These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md). + These files are set up by the Nix installer. See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable. > **Note** @@ -173,7 +184,7 @@ struct EvalSettings : Config R"( If set to `true`, the Nix evaluator doesn't allow access to any files outside of - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). )"}; @@ -302,7 +313,7 @@ struct EvalSettings : Config "ignore-try", R"( If set to true, ignore exceptions inside 'tryEval' calls when evaluating Nix expressions in - debug mode (using the --debugger flag). By default the debugger pauses on all exceptions. + debug mode (using the --debugger flag). By default, the debugger pauses on all exceptions. )"}; Setting traceVerbose{ @@ -320,7 +331,7 @@ struct EvalSettings : Config "debugger-on-trace", R"( If set to true and the `--debugger` flag is given, the following functions - enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break): + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose) @@ -336,7 +347,7 @@ struct EvalSettings : Config "debugger-on-warn", R"( If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) - will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). This is useful for debugging warnings in third-party Nix code. @@ -350,7 +361,7 @@ struct EvalSettings : Config R"( If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) throws an error when logging a warning. - This will give you a stack trace that leads to the location of the warning. + This gives you a stack trace that leads to the location of the warning. This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn). @@ -466,6 +477,44 @@ struct EvalSettings : Config The default value is chosen to balance performance and memory usage. On 32 bit systems where memory is scarce, the default is a large value to reduce the amount of allocations. )"}; + + Setting lazyTrees{ + this, + false, + "lazy-trees", + R"( + If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. + )"}; + + // FIXME: this setting should really be in libflake, but it's + // currently needed in mountInput(). + Setting lazyLocks{ + this, + false, + "lazy-locks", + R"( + If enabled, Nix only includes NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + This is not backward compatible with older versions of Nix. + If disabled, lock file entries always contain a NAR hash. + )"}; + + Setting evalCores{ + this, + 1, + "eval-cores", + R"( + The number of threads used to evaluate Nix expressions. This currently affects the following commands: + + * `nix search` + * `nix flake check` + * `nix flake show` + * `nix eval --json` + * Any evaluation that uses `builtins.parallel` + + The value `0` causes Nix to use all available CPU cores in the system. + + Note that enabling the debugger (`--debugger`) disables multi-threaded evaluation. + )"}; }; /** @@ -473,4 +522,10 @@ struct EvalSettings : Config */ std::filesystem::path getNixDefExpr(); +/** + * Stack size for evaluator threads. This used to be 64 MiB, but macOS as deployed on GitHub Actions has a + * hard limit slightly under that, so we round it down a bit. + */ +constexpr size_t evalStackSize = 60 * 1024 * 1024; + } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index bea616187b41..b0ae6c6ecb0f 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -6,6 +6,7 @@ #include "nix/expr/eval-profiler.hh" #include "nix/util/types.hh" #include "nix/expr/value.hh" +#include "nix/expr/root-value.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/util/configuration.hh" @@ -27,6 +28,7 @@ #include #include #include +#include namespace nix { @@ -51,6 +53,9 @@ struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; struct MountedSourceAccessor; +struct AsyncPathWriter; +struct Provenance; +struct Executor; namespace eval_cache { class EvalCache; @@ -164,9 +169,7 @@ struct Constant bool impureOnly = false; }; -typedef std:: - map, traceable_allocator>> - ValMap; +typedef std::map ValMap; typedef boost::unordered_flat_map> DocCommentMap; @@ -225,7 +228,7 @@ struct StaticEvalSymbols line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure, outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet, - operator_, key, path, prefix, outputSpecified; + operator_, key, path, prefix, outputSpecified, __meta; Expr::AstSymbols exprSymbols; @@ -278,6 +281,7 @@ struct StaticEvalSymbols .path = alloc.create("path"), .prefix = alloc.create("prefix"), .outputSpecified = alloc.create("outputSpecified"), + .__meta = alloc.create("__meta"), .exprSymbols = { .sub = alloc.create("__sub"), .lessThan = alloc.create("__lessThan"), @@ -395,6 +399,7 @@ public: const ref internalFS; const SourcePath derivationInternal; + const SourcePath importedDrvToDerivation; /** * Store used to materialise .drv files. @@ -406,8 +411,6 @@ public: */ const ref buildStore; - RootValue vImportedDrvToDerivation = nullptr; - const ref inputCache; /** @@ -420,6 +423,8 @@ public: std::list debugTraces; boost::unordered_flat_map> exprEnvs; + ref asyncPathWriter; + const std::shared_ptr getStaticEnv(const Expr & expr) const { auto i = exprEnvs.find(&expr); @@ -459,10 +464,6 @@ public: private: - /* Cache for calls to addToStore(); maps source paths to the store - paths. */ - const ref> srcToStore; - /** * A cache that maps paths to "resolved" paths for importing Nix * expressions, i.e. `/foo` to `/foo/default.nix`. @@ -472,23 +473,25 @@ private: /** * A cache from resolved paths to values. */ - const ref, - std::equal_to, - traceable_allocator>>> - fileEvalCache; + const ref> fileEvalCache; /** * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - boost::unordered_flat_map positionToDocComment; + const ref>> positionToDocComment; LookupPath lookupPath; - boost::unordered_flat_map, StringViewHash, std::equal_to<>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -576,7 +579,12 @@ public: /** * Mount an input on the Nix store. */ - StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor); + StorePath mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash = false); /** * Parse a Nix expression from the specified file. @@ -625,9 +633,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -650,25 +659,10 @@ public: * application, call the function and overwrite `v` with the * result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v, const PosIdx pos); - -private: - - /** - * Internal support function for forceValue - * - * This code is factored out so that it's not in the heavily inlined hot path. - */ - void handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos); - - /** - * Internal support function for forceValue - * - * This code is factored out so that it's not in the heavily inlined hot path. - */ - void handleEvalExceptionForApp(Value & v, const Value & savedApp); - - void handleEvalFailed(Value & v, PosIdx pos); + inline void forceValue(Value & v, const PosIdx pos) + { + v.force(*this, pos); + } void tryFixupBlackHolePos(Value & v, PosIdx pos); @@ -728,6 +722,12 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + StorePath devirtualize(const StorePath & path, StringMap * rewrites = nullptr); + + SingleDerivedPath devirtualize(const SingleDerivedPath & path, StringMap * rewrites = nullptr); + + std::string devirtualize(std::string_view s, const NixStringContext & context); + /** * String coercion. * @@ -745,7 +745,19 @@ public: bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos); + + /** + * Compute the base name for a `SourcePath`. For non-store paths, + * this is just `SourcePath::baseName()`. But for store paths, for + * backwards compatibility, it needs to be `-source`, + * i.e. as if the path were copied to the Nix store. This results + * in a "double-copied" store path like + * `/nix/store/--source`. We don't need to + * materialize /nix/store/-source though. Still, this + * requires reading/hashing the path twice. + */ + std::string computeBaseName(const SourcePath & path, PosIdx pos); /** * Path coercion. @@ -808,13 +820,7 @@ public: /** * Internal primops not exposed to the user. */ - boost::unordered_flat_map< - std::string, - Value *, - StringViewHash, - std::equal_to<>, - traceable_allocator>> - internalPrimOps; + boost::unordered_flat_map internalPrimOps; /** * Name and documentation about every constant. @@ -895,10 +901,11 @@ private: const std::shared_ptr & staticEnv); /** - * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run - * out of system stack. + * Current Nix call stack depth, used with `max-call-depth` + * setting to throw stack overflow hopefully before we run out of + * system stack. */ - size_t callDepth = 0; + [[gnu::tls_model("initial-exec")]] thread_local static size_t callDepth; public: @@ -997,7 +1004,10 @@ public: */ void mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v); - void concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx); + /** + * @brief Concatenate values with an n-ary version of the `++` operator. + */ + void concatLists(Value & v, std::span lists, const PosIdx pos, std::string_view errorCtx); /** * Print statistics, if enabled. @@ -1051,6 +1061,10 @@ public: DocComment getDocCommentForPos(PosIdx pos); + void waitForPath(const StorePath & path); + void waitForPath(const SingleDerivedPath & path); + void waitForAllPaths(); + private: /** @@ -1076,21 +1090,30 @@ private: Counter nrPrimOpCalls; Counter nrFunctionCalls; - bool countCalls; +public: + Counter nrThunksAwaited; + Counter nrThunksAwaitedSlow; + Counter microsecondsWaiting; + Counter currentlyWaiting; + Counter maxWaiting; + Counter nrSpuriousWakeups; + +private: + const bool countCalls; - typedef boost::unordered_flat_map> PrimOpCalls; - PrimOpCalls primOpCalls; + typedef boost::concurrent_flat_map> PrimOpCalls; + const ref primOpCalls; - typedef boost::unordered_flat_map FunctionCalls; - FunctionCalls functionCalls; + typedef boost::concurrent_flat_map FunctionCalls; + const ref functionCalls; /** Evaluation/call profiler. */ MultiEvalProfiler profiler; void incrFunctionCall(ExprLambda * fun); - typedef boost::unordered_flat_map> AttrSelects; - AttrSelects attrSelects; + typedef boost::concurrent_flat_map> AttrSelects; + const ref attrSelects; friend struct ExprOpUpdate; friend struct ExprOpConcatLists; @@ -1106,6 +1129,56 @@ private: friend struct Value; friend class ListBuilder; + +public: + + /** + * Per-thread evaluation context. This context is propagated to worker threads when a value is evaluated + * asynchronously. + */ + struct EvalContext + { + std::shared_ptr provenance; + }; + + [[gnu::tls_model("initial-exec")]] thread_local static EvalContext evalContext; + + /** + * Create a work item that propagates the current evaluation context. + */ + template + auto makeWork(T && t) + { + return [this, t{std::move(t)}, evalContext(evalContext)]() { + this->evalContext = evalContext; + t(); + }; + } + + /** + * Add a work item to the given work vector that propagates the current evaluation context. + */ + template + void addWork(WorkItems & work, uint8_t priority, T && t) + { + work.emplace_back(makeWork(std::move(t)), priority); + } + + template + void spawn(FuturesVector & futures, uint8_t priority, T && t) + { + futures.spawn(priority, makeWork(std::move(t))); + } + + /** + * Worker threads manager. + * + * Note: keep this last to ensure that it's destroyed first, so we + * don't have any background work items (e.g. from + * `builtins.parallel`) referring to a partially destroyed + * `EvalState`. + */ + ref executor; }; struct DebugTraceStacker @@ -1142,6 +1215,24 @@ SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true); */ bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); +struct PushProvenance +{ + EvalState & state; + std::shared_ptr prev; + + PushProvenance(EvalState & state, std::shared_ptr prov) + : state(state) + { + state.evalContext.provenance.swap(prev); + state.evalContext.provenance.swap(prov); + } + + ~PushProvenance() + { + state.evalContext.provenance.swap(prev); + } +}; + } // namespace nix #include "nix/expr/eval-inline.hh" diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 000000000000..3eb8a01c0c5f --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/get-drvs.hh b/src/libexpr/include/nix/expr/get-drvs.hh index 4beccabe2ad3..4f655d514ed9 100644 --- a/src/libexpr/include/nix/expr/get-drvs.hh +++ b/src/libexpr/include/nix/expr/get-drvs.hh @@ -32,6 +32,7 @@ private: */ bool failed = false; + // FIXME: make this a RootValue. const Bindings *attrs = nullptr, *meta = nullptr; const Bindings * getMeta(); diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 4213476fe73a..83661f1c7165 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,17 +20,21 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', 'nixexpr.hh', + 'parallel-eval.hh', 'parser-state.hh', 'primops.hh', 'print-ambiguous.hh', 'print-options.hh', 'print.hh', + 'provenance.hh', 'repl-exit-status.hh', + 'root-value.hh', 'search-path.hh', 'static-string-data.hh', 'symbol-table.hh', diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index df39ecdde913..85a6c0c3e417 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -8,7 +8,6 @@ #include #include -#include "nix/expr/gc-small-vector.hh" #include "nix/expr/value.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/eval-error.hh" @@ -17,6 +16,7 @@ #include "nix/expr/counter.hh" #include "nix/util/pos-table.hh" #include "nix/util/error.hh" +#include "nix/util/bump-memory-resource.hh" namespace nix { @@ -91,8 +91,6 @@ typedef std::vector AttrSelectionPath; std::string showAttrSelectionPath(const SymbolTable & symbols, std::span attrPath); -using UpdateQueue = SmallTemporaryValueVector; - /* Abstract syntax of Nix expressions. */ struct Expr @@ -123,14 +121,6 @@ struct Expr * of thunks allocated. */ virtual Value * maybeThunk(EvalState & state, Env & env); - - /** - * Only called when performing an attrset update: `//` or similar. - * Instead of writing to a Value &, this function writes to an UpdateQueue. - * This allows the expression to perform multiple updates in a delayed manner, gathering up all the updates before - * applying them. - */ - virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx); virtual void setName(Symbol name); virtual void setDocComment(DocComment docComment) {}; @@ -556,7 +546,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; @@ -738,7 +728,7 @@ struct ExprOpNot : Expr struct name : Expr \ { \ MakeBinOpMembers(name, s) \ - } + }; MakeBinOp(ExprOpEq, "=="); MakeBinOp(ExprOpNEq, "!="); @@ -749,14 +739,7 @@ MakeBinOp(ExprOpConcatLists, "++"); struct ExprOpUpdate : Expr { -private: - /** Special case for merging of two attrsets. */ - void eval(EvalState & state, Value & v, Value & v1, Value & v2); - void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q); - -public: - MakeBinOpMembers(ExprOpUpdate, "//"); - virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) override; + MakeBinOpMembers(ExprOpUpdate, "//") }; struct ExprConcatStrings : Expr @@ -811,23 +794,12 @@ struct ExprPos : Expr COMMON_METHODS }; -/* only used to mark thunks as black holes. */ -struct ExprBlackHole : Expr -{ - void show(const SymbolTable & symbols, std::ostream & str) const override {} - - void eval(EvalState & state, Env & env, Value & v) override; - - void bindVars(EvalState & es, const std::shared_ptr & env) override {} - - [[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v); -}; - -extern ExprBlackHole eBlackHole; - class Exprs { - std::pmr::monotonic_buffer_resource buffer; + /* Thread-safe fallback resource, which might be a bit slower. */ + std::pmr::synchronized_pool_resource fallbackResource; + BumpMemoryResource buffer{BumpMemoryResource::defaultReserveSize, &fallbackResource}; + public: std::pmr::polymorphic_allocator alloc{&buffer}; diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh new file mode 100644 index 000000000000..4f611328f4ff --- /dev/null +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "nix/util/move-only-function.hh" +#include "nix/util/sync.hh" +#include "nix/util/logging.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/util.hh" +#include "nix/util/signals.hh" + +#if NIX_USE_BOEHMGC +# include +#endif + +namespace nix { + +struct Executor +{ + using work_t = MoveOnlyFunction; + + struct Item + { + std::promise promise; + work_t work; + }; + + struct State + { + std::multimap queue; + std::vector threads; + }; + + std::atomic_bool quit{false}; + + const unsigned int evalCores; + + const bool enabled; + + const std::unique_ptr interruptCallback; + + Sync state_; + + std::condition_variable wakeup; + + static unsigned int getEvalCores(const EvalSettings & evalSettings); + + Executor(const EvalSettings & evalSettings); + + ~Executor(); + + void createWorker(State & state); + + void worker(); + + using WorkItems = std::vector>; + + std::vector> spawn(WorkItems && items); + + [[gnu::tls_model("initial-exec")]] static thread_local bool amWorkerThread; +}; + +struct FutureVector +{ + Executor & executor; + + struct State + { + std::vector> futures; + }; + + Sync state_; + + ~FutureVector(); + + // FIXME: add a destructor that cancels/waits for all futures. + + void spawn(Executor::WorkItems && work); + + void spawn(uint8_t prioPrefix, Executor::work_t && work) + { + Executor::WorkItems items; + items.emplace_back(std::move(work), prioPrefix); + spawn(std::move(items)); + } + + void finishAll(); +}; + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e42..2482d53ea041 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/include/nix/expr/provenance.hh b/src/libexpr/include/nix/expr/provenance.hh new file mode 100644 index 000000000000..f4cc887a6b2a --- /dev/null +++ b/src/libexpr/include/nix/expr/provenance.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "nix/util/provenance.hh" + +namespace nix { + +/** + * Provenance indicating that this store path was instantiated by the `derivation` builtin function. Its main purpose is + * to record `meta` fields. + */ +struct DerivationProvenance : Provenance +{ + std::shared_ptr next; + ref meta; + + DerivationProvenance(std::shared_ptr next, ref meta) + : next(std::move(next)) + , meta(std::move(meta)) {}; + + nlohmann::json to_json() const override; +}; + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/root-value.hh b/src/libexpr/include/nix/expr/root-value.hh new file mode 100644 index 000000000000..7d5fa8767240 --- /dev/null +++ b/src/libexpr/include/nix/expr/root-value.hh @@ -0,0 +1,80 @@ +#pragma once +///@file + +#include +#include + +namespace nix { + +struct Value; + +/** + * A move-only handle rooting a Value, i.e. keeping it and everything + * reachable from it alive across garbage collections. Prefer this + * over `RootValue` unless the handle must be copyable (e.g. when it's + * captured in a `std::function`-backed lambda). + */ +class RootValue +{ + Value ** slot = nullptr; + + /** + * Clear the given slot and return it to the root value pool. + */ + void freeRootValueSlot(); + +public: + RootValue() = default; + + /** + * Allocate a slot from the root value pool, i.e. a GC-visible + * `Value *` cell that keeps the value it points to alive across + * garbage collections. Use `RootValue`/`RootValue` rather than + * calling this directly. + */ + explicit RootValue(Value * v); + + RootValue(const RootValue &) = delete; + RootValue & operator=(const RootValue &) = delete; + + RootValue(RootValue && other) noexcept + : slot(std::exchange(other.slot, nullptr)) + { + } + + RootValue & operator=(RootValue && other) noexcept + { + if (this == &other) + return *this; + if (slot) + freeRootValueSlot(); + slot = std::exchange(other.slot, nullptr); + return *this; + } + + ~RootValue() + { + reset(); + } + + /** + * Release the slot, i.e. stop rooting the value. + */ + void reset() + { + if (slot) + freeRootValueSlot(); + } + + Value *& operator*() const + { + return *slot; + } + + explicit operator bool() const + { + return slot != nullptr; + } +}; + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index f0220376c53f..231510829331 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -2,13 +2,14 @@ ///@file #include + #include "nix/expr/value.hh" -#include "nix/expr/static-string-data.hh" -#include "nix/util/chunked-vector.hh" #include "nix/util/error.hh" +#include "nix/util/sync.hh" +#include "nix/util/alignment.hh" #include -#include +#include namespace nix { @@ -17,17 +18,27 @@ class SymbolValue : protected Value friend class SymbolStr; friend class SymbolTable; - uint32_t idx; - - SymbolValue() = default; - -public: operator std::string_view() const noexcept { return string_view(); } }; +struct ContiguousArena +{ + const char * data; + const size_t maxSize; + + // Put this in a separate cache line to ensure that a thread + // adding a symbol doesn't slow down threads dereferencing symbols + // by invalidating the read-only `data` field. + alignas(64) std::atomic size{0}; + + ContiguousArena(size_t maxSize); + + size_t allocate(size_t bytes); +}; + class StaticSymbolTable; /** @@ -42,6 +53,7 @@ class Symbol friend class StaticSymbolTable; private: + /// The offset of the symbol in `SymbolTable::arena`. uint32_t id; explicit constexpr Symbol(uint32_t id) noexcept @@ -73,6 +85,8 @@ public: constexpr auto operator<=>(const Symbol & other) const noexcept = default; friend class std::hash; + + constexpr static size_t alignment = alignof(SymbolValue); }; /** @@ -84,25 +98,20 @@ class SymbolStr { friend class SymbolTable; - constexpr static size_t chunkSize{8192}; - using SymbolValueStore = ChunkedVector; - const SymbolValue * s; struct Key { using HashType = boost::hash; - SymbolValueStore & store; std::string_view s; std::size_t hash; - std::pmr::memory_resource & resource; + ContiguousArena & arena; - Key(SymbolValueStore & store, std::string_view s, std::pmr::memory_resource & stringMemory) - : store(store) - , s(s) + Key(std::string_view s, ContiguousArena & arena) + : s(s) , hash(HashType{}(s)) - , resource(stringMemory) + , arena(arena) { } }; @@ -113,22 +122,7 @@ public: { } - SymbolStr(const Key & key) - { - auto size = key.s.size(); - if (size >= std::numeric_limits::max()) { - throw Error("Size of symbol exceeds 4GiB and cannot be stored"); - } - // for multi-threaded implementations: lock store and allocator here - const auto & [v, idx] = key.store.add(SymbolValue{}); - if (size == 0) { - v.mkStringNoCopy(""_sds, nullptr); - } else { - v.mkStringNoCopy(StringData::make(key.resource, key.s)); - } - v.idx = idx; - this->s = &v; - } + SymbolStr(const Key & key); bool operator==(std::string_view s2) const noexcept { @@ -157,11 +151,7 @@ public: [[gnu::always_inline]] bool empty() const noexcept { - auto * p = &s->string_data(); - // Save a dereference in the sentinel value case - if (p == &""_sds) - return true; - return p->size() == 0; + return !s->string_data().size(); } [[gnu::always_inline]] @@ -176,11 +166,6 @@ public: return s; } - explicit operator Symbol() const noexcept - { - return Symbol{s->idx + 1}; - } - struct Hash { using is_transparent = void; @@ -218,6 +203,11 @@ public: return operator()(b, a); } }; + + constexpr static size_t computeSize(std::string_view s) + { + return alignUp(sizeof(Value) + sizeof(StringData) + s.size() + 1, Symbol::alignment); + } }; class SymbolTable; @@ -237,6 +227,7 @@ class StaticSymbolTable std::array symbols; std::size_t size = 0; + std::size_t nextId = alignof(SymbolValue); public: constexpr StaticSymbolTable() = default; @@ -245,8 +236,9 @@ public: { /* No need to check bounds because out of bounds access is a compilation error. */ - auto sym = Symbol(size + 1); //< +1 because Symbol with id = 0 is reserved + auto sym = Symbol(nextId); symbols[size++] = {str, sym}; + nextId += SymbolStr::computeSize(str); return sym; } @@ -264,61 +256,67 @@ private: * SymbolTable is an append only data structure. * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based. */ - std::pmr::monotonic_buffer_resource buffer; - SymbolStr::SymbolValueStore store{16}; + ContiguousArena arena; /** - * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store. - * ChunkedVector references are never invalidated. + * Transparent lookup of string view for a pointer to a + * SymbolValue in the arena. */ - boost::unordered_flat_set symbols{SymbolStr::chunkSize}; + boost::concurrent_flat_set symbols; public: SymbolTable(const StaticSymbolTable & staticSymtab) + : arena(1 << 30) { + // Reserve symbol ID 0 and ensure alignment of the first allocation. + arena.allocate(Symbol::alignment); + staticSymtab.copyIntoSymbolTable(*this); } /** * Converts a string into a symbol. */ - Symbol create(std::string_view s) - { - // Most symbols are looked up more than once, so we trade off insertion performance - // for lookup performance. - // FIXME: make this thread-safe. - return Symbol(*symbols.insert(SymbolStr::Key{store, s, buffer}).first); - } + Symbol create(std::string_view s); std::vector resolve(const std::span & symbols) const { std::vector result; result.reserve(symbols.size()); - for (auto sym : symbols) + for (auto & sym : symbols) result.push_back((*this)[sym]); return result; } SymbolStr operator[](Symbol s) const { - uint32_t idx = s.id - uint32_t(1); - if (idx >= store.size()) - unreachable(); - return store[idx]; + assert(s.id); + // Note: we don't check arena.size here to avoid a dependency + // on other threads creating new symbols. + return SymbolStr(*reinterpret_cast(arena.data + s.id)); } - [[gnu::always_inline]] size_t size() const noexcept { - return store.size(); + return symbols.size(); } - size_t totalSize() const; + size_t totalSize() const + { + return arena.size; + } template void dump(T callback) const { - store.forEach(callback); + std::string_view left{arena.data, arena.size}; + left = left.substr(Symbol::alignment); + while (!left.empty()) { + auto v = reinterpret_cast(left.data()); + callback(v->string_view()); + left = left.substr( + alignUp(sizeof(SymbolValue) + sizeof(StringData) + v->string_view().size() + 1, Symbol::alignment)); + } } }; diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index e1b2bc4e25a5..60c29eeddf40 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +29,19 @@ namespace nix { struct Value; class BindingsBuilder; +static constexpr int discriminatorBits = 3; + +enum PrimaryDiscriminator : int { + pdSingleDWord = 0, + pdThunk = 1, + pdPending = 2, + pdAwaited = 3, + pdPairOfPointers = 4, + pdListN = 5, // FIXME: get rid of this by putting the size in the first word + pdString = 6, + pdPath = 7, // FIXME: get rid of this by ditching the `accessor` field +}; + /** * Internal type discriminator, which is more detailed than `ValueType`, as * it specifies the exact representation used (for types that have multiple @@ -36,32 +51,50 @@ class BindingsBuilder; * about how this is mapped into the alignment bits to save significant memory. * This also restricts the number of internal types represented with distinct memory layouts. */ -enum InternalType { - tUninitialized = 0, - /* layout: Single/zero field payload */ - tInt = 1, - tBool, - tNull, - tFloat, - tFailed, - tExternal, - tPrimOp, - tAttrs, - /* layout: Pair of pointers payload */ - tFirstPairOfPointers, - tListSmall = tFirstPairOfPointers, - tPrimOpApp, - tApp, - tThunk, - tLambda, - tLastPairOfPointers = tLambda, - /* layout: Single untaggable field */ - tFirstSingleUntaggable, - tListN = tFirstSingleUntaggable, - tString, - tPath, - tNumberOfInternalTypes, // Must be last -}; +typedef enum { + /* Values that have more type bits in the first word, and the + payload (a single word) in the second word. */ + tUninitialized = PrimaryDiscriminator::pdSingleDWord | (0 << discriminatorBits), + tInt = PrimaryDiscriminator::pdSingleDWord | (1 << discriminatorBits), + tFloat = PrimaryDiscriminator::pdSingleDWord | (2 << discriminatorBits), + tBool = PrimaryDiscriminator::pdSingleDWord | (3 << discriminatorBits), + tNull = PrimaryDiscriminator::pdSingleDWord | (4 << discriminatorBits), + tAttrs = PrimaryDiscriminator::pdSingleDWord | (5 << discriminatorBits), + tPrimOp = PrimaryDiscriminator::pdSingleDWord | (6 << discriminatorBits), + tFailed = PrimaryDiscriminator::pdSingleDWord | (7 << discriminatorBits), + tExternal = PrimaryDiscriminator::pdSingleDWord | (8 << discriminatorBits), + + /* Thunks. */ + tThunk = PrimaryDiscriminator::pdThunk | (0 << discriminatorBits), + tApp = PrimaryDiscriminator::pdThunk | (1 << discriminatorBits), + + tPending = PrimaryDiscriminator::pdPending, + tAwaited = PrimaryDiscriminator::pdAwaited, + + /* Values that consist of two pointers. The second word contains + more type bits in its alignment niche. */ + tListSmall = PrimaryDiscriminator::pdPairOfPointers | (0 << discriminatorBits), + tPrimOpApp = PrimaryDiscriminator::pdPairOfPointers | (1 << discriminatorBits), + tLambda = PrimaryDiscriminator::pdPairOfPointers | (2 << discriminatorBits), + + /* Special values. */ + tListN = PrimaryDiscriminator::pdListN, + tString = PrimaryDiscriminator::pdString, + tPath = PrimaryDiscriminator::pdPath, +} InternalType; + +/** + * Return true if `type` denotes a "finished" value, i.e. a weak-head + * normal form. + * + * Note that tPrimOpApp is considered "finished" because it represents + * a primop call with an incomplete number of arguments, and therefore + * cannot be evaluated further. + */ +inline bool isFinished(InternalType t) +{ + return t != tUninitialized && t != tThunk && t != tApp && t != tPending && t != tAwaited; +} /** * This type abstracts over all actual value types in the language, @@ -87,7 +120,6 @@ class Bindings; struct Env; struct Expr; struct ExprLambda; -struct ExprBlackHole; struct PrimOp; class Symbol; class SymbolStr; @@ -292,7 +324,7 @@ namespace detail { /** * Implementation mixin class for defining the public types - * In can be inherited from by the actual ValueStorage implementations + * In can be inherited by the actual ValueStorage implementations * for free due to Empty Base Class Optimization (EBCO). */ struct ValueBase @@ -428,34 +460,9 @@ struct ValueBase Value * const * elems; }; - struct Failed : gc_cleanup + struct Failed : gc { std::exception_ptr ex; - /** - * Optional value for recovering `RecoverableEvalError` - * Must be set iff `ex` is an instance of `RecoverableEvalError`. - */ - Value * recoveryValue; - - Failed(std::exception_ptr ex, Value * recoveryValue) - : ex(ex) - , recoveryValue(recoveryValue) - { - } - - [[noreturn]] void rethrow() const - { - try { - std::rethrow_exception(ex); - } catch (BaseError & e) { - /* Rethrow the copy of the exception - not the original one. - Stack tracing mechanisms rely on being able to modify the exceptions - they catch by reference. */ - e.throwClone(); - } catch (...) { - throw; - } - } }; }; @@ -587,12 +594,44 @@ class alignas(16) }; using PackedPointer = typename PackedPointerTypeStruct::type; - using Payload = std::array; - Payload payload = {}; - static constexpr int discriminatorBits = 3; + /** + * For multithreaded evaluation, we have to make sure that thunks/apps + * (the only mutable types of values) are updated in a safe way. A + * value can have the following states (see `force()`): + * + * * "thunk"/"app". When forced, this value transitions to + * "pending". The current thread will evaluate the + * thunk/app. When done, it will override the value with the + * result. If the value is at that point in the "awaited" state, + * the thread will wake up any waiting threads. + * + * * "pending". This means it's currently being evaluated. If + * another thread forces this value, it transitions to "awaited" + * and the thread will wait for the value to be updated (see + * `waitOnThunk()`). + * + * * "awaited". Like pending, only it means that there already are + * one or more threads waiting for this thunk. + * + * To ensure race-free access, the non-atomic word `p1` must + * always be updated before `p0`. Writes to `p0` should use + * *release* semantics (so that `p1` and any referenced values become + * visible to threads that read `p0`), and reads from `p0` should + * use `*acquire* semantics. + * + * Note: at some point, we may want to switch to 128-bit atomics + * so that `p0` and `p1` can be updated together + * atomically. However, 128-bit atomics are a bit problematic at + * present on x86_64 (see + * e.g. https://ibraheem.ca/posts/128-bit-atomics/). + */ + std::atomic p0{0}; + PackedPointer p1{0}; + static constexpr PackedPointer discriminatorMask = (PackedPointer(1) << discriminatorBits) - 1; + // FIXME: move/update /** * The value is stored as a pair of 8-byte double words. All pointers are assumed * to be 8-byte aligned. This gives us at most 6 bits of discriminator bits @@ -622,15 +661,6 @@ class alignas(16) * The primary discriminator with value 0 is reserved for uninitialized Values, * which are useful for diagnostics in C bindings. */ - enum PrimaryDiscriminator : int { - pdUninitialized = 0, - pdSingleDWord, //< layout: Single/zero field payload - /* The order of these enumerations must be the same as in InternalType. */ - pdListN, //< layout: Single untaggable field. - pdString, - pdPath, - pdPairOfPointers, //< layout: Pair of pointers payload - }; template requires std::is_pointer_v @@ -641,7 +671,7 @@ class alignas(16) PrimaryDiscriminator getPrimaryDiscriminator() const noexcept { - return static_cast(payload[0] & discriminatorMask); + return static_cast(p0 & discriminatorMask); } static void assertAligned(PackedPointer val) noexcept @@ -649,13 +679,30 @@ class alignas(16) assert((val & discriminatorMask) == 0 && "Pointer is not 8 bytes aligned"); } + void finish(PackedPointer p0_, PackedPointer p1_) + { + // Note: p1 *must* be updated before p0. + p1 = p1_; + p0_ = p0.exchange(p0_, std::memory_order_release); + + auto pd = static_cast(p0_ & discriminatorMask); + if (pd == pdPending) + // Nothing to do; no thread is waiting on this thunk. + ; + else if (pd == pdAwaited) + // Slow path: wake up the threads that are waiting on this + // thunk. + notifyWaiters(); + else if (pd == pdThunk) + unreachable(); + } + template void setSingleDWordPayload(PackedPointer untaggedVal) noexcept { - /* There's plenty of free upper bits in the first dword, which is - used only for the discriminator. */ - payload[0] = static_cast(pdSingleDWord) | (static_cast(type) << discriminatorBits); - payload[1] = untaggedVal; + /* There's plenty of free upper bits in the first byte, which + is used only for the discriminator. */ + finish(static_cast(type), untaggedVal); } template @@ -664,32 +711,42 @@ class alignas(16) static_assert(discriminator >= pdListN && discriminator <= pdPath); auto firstFieldPayload = std::bit_cast(firstPtrField); assertAligned(firstFieldPayload); - payload[0] = static_cast(discriminator) | firstFieldPayload; - payload[1] = std::bit_cast(untaggableField); + finish(static_cast(discriminator) | firstFieldPayload, std::bit_cast(untaggableField)); } template void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept { - static_assert(type >= tFirstPairOfPointers && type <= tLastPairOfPointers); - { - auto firstFieldPayload = std::bit_cast(firstPtrField); - assertAligned(firstFieldPayload); - payload[0] = static_cast(pdPairOfPointers) | firstFieldPayload; - } - { - auto secondFieldPayload = std::bit_cast(secondPtrField); - assertAligned(secondFieldPayload); - payload[1] = (type - tFirstPairOfPointers) | secondFieldPayload; - } + static_assert(type >= tListSmall && type <= tLambda); + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + auto secondFieldPayload = std::bit_cast(secondPtrField); + assertAligned(secondFieldPayload); + finish( + static_cast(pdPairOfPointers) | firstFieldPayload, + ((type - tListSmall) >> discriminatorBits) | secondFieldPayload); + } + + template + void setThunkPayload(T * firstPtrField, U * secondPtrField) noexcept + { + static_assert(type >= tThunk && type <= tApp); + auto secondFieldPayload = std::bit_cast(secondPtrField); + assertAligned(secondFieldPayload); + p1 = ((type - tThunk) >> discriminatorBits) | secondFieldPayload; + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + // Note: awaited values can never become a thunk, so no need + // to check for waiters. + p0.store(static_cast(pdThunk) | firstFieldPayload, std::memory_order_release); } template requires std::is_pointer_v && std::is_pointer_v void getPairOfPointersPayload(T & firstPtrField, U & secondPtrField) const noexcept { - firstPtrField = untagPointer(payload[0]); - secondPtrField = untagPointer(payload[1]); + firstPtrField = untagPointer(p0); + secondPtrField = untagPointer(p1); } protected: @@ -697,42 +754,45 @@ protected: InternalType getInternalType() const noexcept { switch (auto pd = getPrimaryDiscriminator()) { - case pdUninitialized: - /* Discriminator value of zero is used to distinguish uninitialized values. */ - return tUninitialized; case pdSingleDWord: - /* Payloads that only use up a single double word store the InternalType - in the upper bits of the first double word. */ - return InternalType(payload[0] >> discriminatorBits); + /* Payloads that only use up a single double word store + the full InternalType in the first byte. */ + return InternalType(p0 & 0xff); + case pdThunk: + return static_cast(tThunk + ((p1 & discriminatorMask) << discriminatorBits)); + case pdPending: + return tPending; + case pdAwaited: + return tAwaited; + case pdPairOfPointers: + return static_cast(tListSmall + ((p1 & discriminatorMask) << discriminatorBits)); /* The order must match that of the enumerations defined in InternalType. */ case pdListN: case pdString: case pdPath: - return static_cast(tFirstSingleUntaggable + (pd - pdListN)); - case pdPairOfPointers: - return static_cast(tFirstPairOfPointers + (payload[1] & discriminatorMask)); + return static_cast(tListN + (pd - pdListN)); [[unlikely]] default: nixUnreachableWhenHardened(); } } -#define NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(TYPE, MEMBER_A, MEMBER_B) \ - \ - void getStorage(TYPE & val) const noexcept \ - { \ - getPairOfPointersPayload(val MEMBER_A, val MEMBER_B); \ - } \ - \ - void setStorage(TYPE val) noexcept \ - { \ - setPairOfPointersPayload>(val MEMBER_A, val MEMBER_B); \ +#define NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(TYPE, SET, MEMBER_A, MEMBER_B) \ + \ + void getStorage(TYPE & val) const noexcept \ + { \ + getPairOfPointersPayload(val MEMBER_A, val MEMBER_B); \ + } \ + \ + void setStorage(TYPE val) noexcept \ + { \ + SET>(val MEMBER_A, val MEMBER_B); \ } - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(SmallList, [0], [1]) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(PrimOpApplicationThunk, .left, .right) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(FunctionApplicationThunk, .left, .right) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(ClosureThunk, .env, .expr) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(Lambda, .env, .fun) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(SmallList, setPairOfPointersPayload, [0], [1]) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(PrimOpApplicationThunk, setPairOfPointersPayload, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(Lambda, setPairOfPointersPayload, .env, .fun) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(FunctionApplicationThunk, setThunkPayload, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(ClosureThunk, setThunkPayload, .env, .expr) #undef NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS @@ -740,57 +800,57 @@ protected: { /* PackedPointerType -> int64_t here is well-formed, since the standard requires this conversion to follow 2's complement rules. This is just a no-op. */ - integer = NixInt(payload[1]); + integer = NixInt(p1); } void getStorage(bool & boolean) const noexcept { - boolean = payload[1]; + boolean = p1; } void getStorage(Null & null) const noexcept {} void getStorage(NixFloat & fpoint) const noexcept { - fpoint = std::bit_cast(payload[1]); + fpoint = std::bit_cast(p1); } void getStorage(ExternalValueBase *& external) const noexcept { - external = std::bit_cast(payload[1]); + external = std::bit_cast(p1); } void getStorage(PrimOp *& primOp) const noexcept { - primOp = std::bit_cast(payload[1]); + primOp = std::bit_cast(p1); } void getStorage(Bindings *& attrs) const noexcept { - attrs = std::bit_cast(payload[1]); + attrs = std::bit_cast(p1); } void getStorage(List & list) const noexcept { - list.elems = untagPointer(payload[0]); - list.size = payload[1]; + list.elems = untagPointer(p0); + list.size = p1; } void getStorage(StringWithContext & string) const noexcept { - string.context = untagPointer(payload[0]); - string.str = std::bit_cast(payload[1]); + string.context = untagPointer(p0); + string.str = std::bit_cast(p1); } void getStorage(Path & path) const noexcept { - path.accessor = untagPointer(payload[0]); - path.path = std::bit_cast(payload[1]); + path.accessor = untagPointer(p0); + path.path = std::bit_cast(p1); } void getStorage(Failed *& failed) const noexcept { - failed = std::bit_cast(payload[1]); + failed = std::bit_cast(p1); } void setStorage(NixInt integer) noexcept @@ -847,8 +907,80 @@ protected: { setSingleDWordPayload(std::bit_cast(failed)); } + + ValueStorage() {} + + ValueStorage(const ValueStorage & v) + { + *this = v; + } + + /** + * Copy a value. This is not allowed to be a thunk to avoid + * accidental work duplication. + */ + ValueStorage & operator=(const ValueStorage & v) + { + auto p0_ = v.p0.load(std::memory_order_acquire); + auto p1_ = v.p1; // must be loaded after p0 + auto pd = static_cast(p0_ & discriminatorMask); + if (pd == pdThunk || pd == pdPending || pd == pdAwaited) + unreachable(); + finish(p0_, p1_); + return *this; + } + +public: + + /** + * Check whether forcing this value requires a trivial amount of + * computation. A value is trivial if it's finished or if it's a + * thunk whose expression is an attrset with no dynamic + * attributes, a lambda or a list. Note that it's up to the caller + * to check whether the members of those attrsets or lists must be + * trivial. + */ + bool isTrivial() const; + + inline void reset() + { + p1 = 0; + p0.store(0, std::memory_order_relaxed); + } + + /// Only used for testing. + inline void mkBlackhole() + { + p0.store(pdPending, std::memory_order_relaxed); + } + + void force(EvalState & state, PosIdx pos); + +private: + + /** + * Given a thunk that was observed to be in the pending or awaited + * state, wait for it to finish. Returns the first word of the + * value. + */ + PackedPointer waitOnThunk(EvalState & state, PackedPointer p0); + + /** + * Wake up any threads that are waiting on this value. + */ + void notifyWaiters(); }; +template<> +void ValueStorage::notifyWaiters(); + +template<> +ValueStorage::PackedPointer +ValueStorage::waitOnThunk(EvalState & state, PackedPointer p0); + +template<> +bool ValueStorage::isTrivial() const; + /** * View into a list of Value * that is itself immutable. * @@ -1090,11 +1222,16 @@ public: void print(EvalState & state, std::ostream & str, PrintOptions options = PrintOptions{}); + // FIXME: optimize, only look at first word + inline bool isFinished() const + { + return nix::isFinished(getInternalType()); + } + // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type - // type() == nThunk inline bool isThunk() const { return isa(); @@ -1105,7 +1242,11 @@ public: return isa(); } - inline bool isBlackhole() const; + inline bool isBlackhole() const + { + auto t = getInternalType(); + return t == tPending || t == tAwaited; + } // type() == nFunction inline bool isLambda() const @@ -1131,17 +1272,13 @@ public: /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd - * - * @param invalidIsThunk Instead of UB an an invalid (probably - * 0, so uninitialized) internal type, return `nThunk`. */ - template inline ValueType type() const { /* Explicit lookup table. switch() might compile down (and it does at least with GCC 14) to a jump table. Let's help the compiler a bit here. */ static constexpr auto table = [] { - std::array t{}; + std::array t{}; t[tUninitialized] = nThunk; t[tInt] = nInt; t[tBool] = nBool; @@ -1162,15 +1299,7 @@ public: return t; }(); - auto it = getInternalType(); - if (it == tUninitialized || it >= tNumberOfInternalTypes) [[unlikely]] { - if constexpr (invalidIsThunk) - return nThunk; - else - nixUnreachableWhenHardened(); - } - - return table[it]; + return table[getInternalType()]; } /** @@ -1261,8 +1390,6 @@ public: setStorage(Lambda{.env = e, .fun = f}); } - inline void mkBlackhole(); - void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) noexcept @@ -1285,9 +1412,9 @@ public: setStorage(n); } - inline void mkFailed(std::exception_ptr e, Value * recovery) noexcept + inline void mkFailed() noexcept { - setStorage(new Value::Failed(e, recovery)); + setStorage(new Value::Failed{.ex = std::current_exception()}); } bool isList() const noexcept @@ -1307,13 +1434,6 @@ public: PosIdx determinePos(const PosIdx pos) const; - /** - * Check whether forcing this value requires a trivial amount of - * computation. In particular, function applications are - * non-trivial. - */ - bool isTrivial() const; - SourcePath path() const { return SourcePath( @@ -1375,6 +1495,7 @@ public: return getStorage(); } + // FIXME: remove this since reading it is racy. ClosureThunk thunk() const noexcept { return getStorage(); @@ -1385,6 +1506,7 @@ public: return getStorage(); } + // FIXME: remove this since reading it is racy. FunctionApplicationThunk app() const noexcept { return getStorage(); @@ -1405,26 +1527,12 @@ public: return getStorage().accessor; } - Failed & failed() const noexcept + Failed * failed() const noexcept { - auto p = getStorage(); - assert(p); - return *p; + return getStorage(); } }; -extern ExprBlackHole eBlackHole; - -bool Value::isBlackhole() const -{ - return isThunk() && thunk().expr == (Expr *) &eBlackHole; -} - -void Value::mkBlackhole() -{ - mkThunk(nullptr, (Expr *) &eBlackHole); -} - typedef std::vector> ValueVector; typedef boost::unordered_flat_map< Symbol, @@ -1433,15 +1541,6 @@ typedef boost::unordered_flat_map< std::equal_to, traceable_allocator>> ValueMap; -typedef std::map, traceable_allocator>> - ValueVectorMap; - -/** - * A value allocated in traceable memory. - */ -typedef std::shared_ptr RootValue; - -RootValue allocRootValue(Value * v); void forceNoNullByte(std::string_view s, std::function = nullptr); } // namespace nix diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index 31f03addf2a0..29cf9ca0b841 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -11,6 +11,8 @@ namespace nix { class BadNixStringContextElem final : public CloneableError { + void anchor() override; + public: std::string_view raw; @@ -64,7 +66,31 @@ struct NixStringContextElem */ using Built = SingleDerivedPath::Built; - using Raw = std::variant; + /** + * A store path that will not result in a store reference when + * used in a derivation or toFile. + * + * When you apply `builtins.toString` to a path value representing + * a path in the Nix store (as is the case with flake inputs), + * historically you got a string without context + * (e.g. `/nix/store/...-source`). This is broken, since it allows + * you to pass a store path to a derivation/toFile without a + * proper store reference. This is especially a problem with lazy + * trees, since the store path is a virtual path that doesn't + * exist. + * + * For backwards compatibility, and to warn users about this + * unsafe use of `toString`, we keep track of such strings as a + * special type of context. + */ + struct Path + { + StorePath storePath; + + GENERATE_CMP(Path, me->storePath); + }; + + using Raw = std::variant; Raw raw; @@ -100,4 +126,10 @@ struct NixStringContextElem */ typedef std::set NixStringContext; +/** + * Returns false if `context` has no elements other than + * `NixStringContextElem::Path`. + */ +bool hasContext(const NixStringContext & context); + } // namespace nix diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 4a68308c6416..378a046687b3 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -3,13 +3,14 @@ #include "nix/expr/eval.hh" #include -#include #include using json = nlohmann::json; namespace nix { +namespace { + // for more information, refer to // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp class JSONSax : nlohmann::json_sax @@ -31,7 +32,7 @@ class JSONSax : nlohmann::json_sax } explicit JSONState(Value * v) - : v(allocRootValue(v)) + : v(RootValue(v)) { } @@ -40,7 +41,7 @@ class JSONSax : nlohmann::json_sax Value & value(EvalState & state) { if (!v) - v = allocRootValue(state.allocValue()); + v = RootValue(state.allocValue()); return **v; } @@ -65,7 +66,7 @@ class JSONSax : nlohmann::json_sax void add() override { - v = nullptr; + v.reset(); } public: void key(string_t & name, EvalState & state) @@ -91,7 +92,7 @@ class JSONSax : nlohmann::json_sax void add() override { values.push_back(*v); - v = nullptr; + v.reset(); } public: JSONListState(std::unique_ptr && p, std::size_t reserve) @@ -201,6 +202,8 @@ class JSONSax : nlohmann::json_sax } }; +} // namespace + void parseJSON(EvalState & state, const std::string_view & s_, Value & v) { JSONSax parser(state, v); @@ -209,4 +212,6 @@ void parseJSON(EvalState & state, const std::string_view & s_, Value & v) throw JSONParseError("Invalid JSON Value"); } +void JSONParseError::anchor() {} + } // namespace nix diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 59f6f6f70dfd..085ac7df4698 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -26,3 +26,5 @@ void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc->beginOffset = loc->endOffset; loc->endOffset += len; } + +nix::Parser::~Parser() {} diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5bdb5335b841..477eee1df9e6 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -33,6 +33,8 @@ namespace nix { struct LexerState; } +// OK because we keep this in a separate compilation unit even for unity +// builds. using namespace nix; using namespace nix::lexer::internal; diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 0b594dc0cf1f..995202dbcb30 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -43,6 +43,7 @@ boost = dependency( modules : [ 'container', 'context', + 'thread', ], include_type : 'system', ) @@ -62,7 +63,6 @@ bdw_gc = dependency('bdw-gc', required : bdw_gc_required) if bdw_gc.found() deps_public += bdw_gc foreach funcspec : [ - 'pthread_attr_get_np', 'pthread_getattr_np', ] define_name = 'HAVE_' + funcspec.underscorify().to_upper() @@ -77,6 +77,17 @@ endif # Used in public header. Affects ABI! configdata_pub.set('NIX_USE_BOEHMGC', bdw_gc.found().to_int()) +link_args = [] + +wasmtime_required = get_option('wasm').disable_if( + get_option('default_library') == 'static', + error_message : 'Building with wasmtime and static linking is not supported', +) + +if wasmtime_required.enabled() + link_args += '-lwasmtime' +endif + toml11 = dependency( 'toml11', version : '>=3.7.0', @@ -164,13 +175,16 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', - 'lexer-helpers.cc', 'nixexpr.cc', + 'parallel-eval.cc', 'paths.cc', 'primops.cc', 'print-ambiguous.cc', 'print.cc', + 'provenance.cc', + 'root-value.cc', 'search-path.cc', + 'symbol-table.cc', 'value-to-json.cc', 'value-to-xml.cc', 'value.cc', @@ -215,6 +229,7 @@ parser_library = static_library( 'nixexpr-parser', parser_tab, lexer_tab, + 'lexer-helpers.cc', cpp_args : parser_library_cpp_args, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, @@ -225,6 +240,8 @@ parser_library = static_library( override_options : [ 'b_ndebug=@0@'.format(not get_option('debug')), 'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'), + # Unity builds mess up the recursive header dependency between flex and bison generated code. + 'unity=off', ], ) @@ -235,12 +252,11 @@ this_library = library( parser_tab[1], lexer_tab[1], generated_headers, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, - link_args : linker_export_flags, + link_args : linker_export_flags + link_args, link_whole : [ parser_library ], - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) diff --git a/src/libexpr/meson.options b/src/libexpr/meson.options index 847bb211d302..2defbf77aaa0 100644 --- a/src/libexpr/meson.options +++ b/src/libexpr/meson.options @@ -3,3 +3,9 @@ option( type : 'feature', description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)', ) + +option( + 'wasm', + type : 'feature', + description : 'enable wasmtime integration into the Nix expression evaluator', +) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4a2f71a11b8f..b52370816f5a 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -13,8 +13,6 @@ namespace nix { Counter Expr::nrExprs; -ExprBlackHole eBlackHole; - // FIXME: remove, because *symbols* are abstract and do not have a single // textual representation; see printIdentifier() std::ostream & operator<<(std::ostream & str, const SymbolStr & symbol) @@ -626,15 +624,6 @@ void ExprLambda::setDocComment(DocComment docComment) // belongs in the same conditional. body->setDocComment(docComment); } -}; - -/* Symbol table. */ - -size_t SymbolTable::totalSize() const -{ - size_t n = 0; - dump([&](SymbolStr s) { n += s.size(); }); - return n; } std::string DocComment::getInnerText(const PosTable & positions) const diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d0aef34e95de..6eccc0645132 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -14,6 +14,7 @@ boehmgc, nlohmann_json, toml11, + wasmtime, # Configuration Options @@ -29,6 +30,11 @@ # Temporarily disabled on Windows because the `GC_throw_bad_alloc` # symbol is missing during linking. enableGC ? !stdenv.hostPlatform.isWindows, + + # Whether to use wasmtime for wasm integration in the Nix language evaluator + # + # Temporarily disabled when static linking due to Rust not compiling + enableWasm ? !stdenv.hostPlatform.isStatic, }: let @@ -36,7 +42,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr"; + pname = "determinate-nix-expr"; inherit version; workDir = ./.; @@ -64,7 +70,8 @@ mkMesonLibrary (finalAttrs: { buildInputs = [ toml11 - ]; + ] + ++ lib.optional enableWasm wasmtime; propagatedBuildInputs = [ nix-util @@ -77,8 +84,20 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ (lib.mesonEnable "gc" enableGC) + (lib.mesonEnable "wasm" enableWasm) ]; + # Fixes a problem with the "nix-expr-libcxxStdenv-static" package output. + # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. + # Force this to be built with compiler-rt over libgcc_eh works. + # Issue: https://github.com/NixOS/nixpkgs/issues/177129 + NIX_CFLAGS_COMPILE = lib.optionalString ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) "-rtlib=compiler-rt"; + meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc new file mode 100644 index 000000000000..1c89b9c4db2c --- /dev/null +++ b/src/libexpr/parallel-eval.cc @@ -0,0 +1,317 @@ +#include "nix/expr/eval.hh" +#include "nix/expr/parallel-eval.hh" +#include "nix/store/globals.hh" +#include "nix/expr/primops.hh" + +namespace nix { + +// cache line alignment to prevent false sharing +struct alignas(64) WaiterDomain +{ + std::condition_variable cv; +}; + +static std::array, 128> waiterDomains; + +[[gnu::tls_model("initial-exec")]] thread_local bool Executor::amWorkerThread{false}; + +unsigned int Executor::getEvalCores(const EvalSettings & evalSettings) +{ + /* Note: the default number of cores is currently limited to 32 + due to scalability bottlenecks. */ + return evalSettings.evalProfilerMode != EvalProfilerMode::disabled ? 1 + : evalSettings.evalCores == 0UL ? std::min(32U, Settings::getDefaultCores()) + : evalSettings.evalCores; +} + +Executor::Executor(const EvalSettings & evalSettings) + : evalCores(getEvalCores(evalSettings)) + , enabled(evalCores > 1) + , interruptCallback(createInterruptCallback([&]() { + for (auto & domain : waiterDomains) + domain.lock()->cv.notify_all(); + })) +{ + debug("executor using %d threads", evalCores); + auto state(state_.lock()); + // FIXME: create worker threads on demand? + for (size_t n = 0; n < evalCores; ++n) + try { + createWorker(*state); + } catch (boost::thread_resource_error & e) { + if (n == 0) + throw Error("could not create any evaluator worker threads: %s", e.what()); + warn("could only create %d evaluator worker threads: %s", n, e.what()); + break; + } +} + +Executor::~Executor() +{ + std::vector threads; + { + auto state(state_.lock()); + quit = true; + std::swap(threads, state->threads); + debug("executor shutting down with %d items left", state->queue.size()); + } + + wakeup.notify_all(); + + for (auto & thr : threads) + thr.join(); +} + +void Executor::createWorker(State & state) +{ + boost::thread::attributes attrs; + attrs.set_stack_size(evalStackSize); + state.threads.push_back(boost::thread(attrs, [&]() { +#if NIX_USE_BOEHMGC + GC_stack_base sb; + GC_get_stack_base(&sb); + GC_register_my_thread(&sb); +#endif + worker(); +#if NIX_USE_BOEHMGC + GC_unregister_my_thread(); +#endif + })); +} + +void Executor::worker() +{ + ReceiveInterrupts receiveInterrupts; + + unix::interruptCheck = [&]() { return (bool) quit; }; + + amWorkerThread = true; + + while (true) { + Item item; + + while (true) { + auto state(state_.lock()); + if (quit) { + // Set an `Interrupted` exception on all promises so + // we get a nicer error than "std::future_error: + // Broken promise". + auto ex = std::make_exception_ptr(Interrupted("interrupted by the user")); + for (auto & item : state->queue) + item.second.promise.set_exception(ex); + state->queue.clear(); + return; + } + if (!state->queue.empty()) { + item = std::move(state->queue.begin()->second); + state->queue.erase(state->queue.begin()); + break; + } + state.wait(wakeup); + } + + try { + item.work(); + item.promise.set_value(); + } catch (const Interrupted &) { + quit = true; + item.promise.set_exception(std::current_exception()); + } catch (...) { + item.promise.set_exception(std::current_exception()); + } + } +} + +std::vector> Executor::spawn(WorkItems && items) +{ + if (items.empty()) + return {}; + + std::vector> futures; + + { + auto state(state_.lock()); + for (auto & item : items) { + std::promise promise; + futures.push_back(promise.get_future()); + /* Note: this uses a cheap PRNG rather than std::random_device, + since the latter costs hundreds of cycles per call (RDRAND or + /dev/urandom), which adds up when spawning many work items. The + key only needs to spread items of the same priority around the + queue, not be cryptographically random. */ + [[gnu::tls_model("initial-exec")]] static thread_local std::mt19937_64 rng{std::random_device{}()}; + [[gnu::tls_model("initial-exec")]] static thread_local std::uniform_int_distribution dist( + 0, 1ULL << 48); + auto key = (uint64_t(item.second) << 48) | dist(rng); + state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(item.first)}); + } + } + + if (items.size() == 1) + wakeup.notify_one(); + else + wakeup.notify_all(); + + return futures; +} + +FutureVector::~FutureVector() +{ + try { + finishAll(); + } catch (...) { + ignoreExceptionInDestructor(); + } +} + +void FutureVector::spawn(Executor::WorkItems && work) +{ + auto futures = executor.spawn(std::move(work)); + auto state(state_.lock()); + for (auto & future : futures) + state->futures.push_back(std::move(future)); +} + +void FutureVector::finishAll() +{ + std::exception_ptr ex; + while (true) { + std::vector> futures; + { + auto state(state_.lock()); + std::swap(futures, state->futures); + } + debug("got %d futures", futures.size()); + if (futures.empty()) + break; + for (auto & future : futures) + try { + future.get(); + } catch (...) { + if (ex) { + if (!getInterrupted()) + logExceptionExceptInterrupt(); + } else + ex = std::current_exception(); + } + } + if (ex) + std::rethrow_exception(ex); +} + +static Sync & getWaiterDomain(detail::ValueBase & v) +{ + auto domain = (((size_t) &v) >> 5) % waiterDomains.size(); + return waiterDomains[domain]; +} + +static std::atomic nextEvalThreadId{1}; +[[gnu::tls_model("initial-exec")]] thread_local uint32_t myEvalThreadId(nextEvalThreadId++); + +template<> +ValueStorage::PackedPointer +ValueStorage::waitOnThunk(EvalState & state, PackedPointer expectedP0) +{ + state.nrThunksAwaited++; + + auto domain = getWaiterDomain(*this).lock(); + + auto threadId = expectedP0 >> discriminatorBits; + + if (static_cast(expectedP0 & discriminatorMask) == pdAwaited) { + /* Make sure that the value is still awaited, now that we're + holding the domain lock. */ + auto p0_ = p0.load(std::memory_order_acquire); + auto pd = static_cast(p0_ & discriminatorMask); + + /* If the value has been finalized in the meantime (i.e. is no + longer pending), we're done. */ + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + return p0_; + } + } else { + /* Mark this value as being waited on. */ + PackedPointer p0_ = expectedP0; + if (!p0.compare_exchange_strong( + p0_, + pdAwaited | (threadId << discriminatorBits), + std::memory_order_acquire, + std::memory_order_acquire)) { + /* If the value has been finalized in the meantime (i.e. is + no longer pending), we're done. */ + auto pd = static_cast(p0_ & discriminatorMask); + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + return p0_; + } + /* The value was already in the "waited on" state, so we're + not the only thread waiting on it. */ + } + } + + /* Wait for another thread to finish this value. */ + if (threadId == myEvalThreadId) + state.error("infinite recursion encountered") + .atPos(((Value &) *this).determinePos(noPos)) + .debugThrow(); + + state.nrThunksAwaitedSlow++; + state.currentlyWaiting++; + state.maxWaiting = std::max(state.maxWaiting, state.currentlyWaiting); + + auto now1 = std::chrono::steady_clock::now(); + + while (true) { + domain.wait(domain->cv); + auto p0_ = p0.load(std::memory_order_acquire); + auto pd = static_cast(p0_ & discriminatorMask); + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + auto now2 = std::chrono::steady_clock::now(); + state.microsecondsWaiting += std::chrono::duration_cast(now2 - now1).count(); + state.currentlyWaiting--; + return p0_; + } + state.nrSpuriousWakeups++; + checkInterrupt(); + } +} + +template<> +void ValueStorage::notifyWaiters() +{ + auto domain = getWaiterDomain(*this).lock(); + + domain->cv.notify_all(); +} + +static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.parallel"); + + if (state.executor->enabled) { + Executor::WorkItems work; + for (auto value : args[0]->listView()) + if (!value->isFinished()) + state.addWork(work, 0, [value(RootValue(value)), &state, pos]() { state.forceValue(**value, pos); }); + state.executor->spawn(std::move(work)); + } + + state.forceValue(*args[1], pos); + v = *args[1]; +} + +// FIXME: gate this behind an experimental feature. +static RegisterPrimOp r_parallel({ + .name = "__parallel", + .args = {"xs", "x"}, + .arity = 2, + .doc = R"( + Start evaluation of the values `xs` in the background and return `x`. + )", + .impl = prim_parallel, + .experimentalFeature = Xp::ParallelEval, +}); + +} // namespace nix diff --git a/src/libexpr/parser-scanner-decls.hh b/src/libexpr/parser-scanner-decls.hh index e4e06188334c..1bec23837b99 100644 --- a/src/libexpr/parser-scanner-decls.hh +++ b/src/libexpr/parser-scanner-decls.hh @@ -11,7 +11,9 @@ namespace nix { class Parser : public parser::BisonParser { +public: using BisonParser::BisonParser; + ~Parser() override; }; } // namespace nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e5e4241ea7b0..77e63ed6299f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -426,7 +426,7 @@ path_start /* Absolute paths are always interpreted relative to the root filesystem accessor, rather than the accessor of the current Nix expression. */ - auto path = canonPath(literal).string(); + auto path = CanonPath(literal).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; @@ -442,8 +442,7 @@ path_start return std::nullopt; }); - auto basePath = std::filesystem::path(state->basePath.path.abs()); - auto path = absPath(literal, &basePath).string(); + auto path = CanonPath(literal, state->basePath.path).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 08c2a45abb0f..6d644fe961e8 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -12,6 +12,8 @@ SourcePath EvalState::rootPath(CanonPath path) SourcePath EvalState::rootPath(std::string_view path) { + /* FIXME: Move this out of EvalState, since it's using native + std::filesystem::path and current working directory. */ return {rootFS, CanonPath(absPath(path).string())}; } @@ -20,24 +22,97 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } -StorePath -EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) +StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) { - auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, + path.name()); + assert(storePath.name() == path.name()); + if (rewrites) + rewrites->emplace(path.hashPart(), storePath.hashPart()); + return storePath; + } else + return path; +} + +SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites) +{ + if (auto o = std::get_if(&path.raw())) + return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)}; + else + return path; +} + +std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context) +{ + StringMap rewrites; + + for (auto & c : context) + if (auto o = std::get_if(&c.raw)) + devirtualize(o->path, &rewrites); + + return rewriteStrings(std::string(s), rewrites); +} + +std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos) +{ + if (path.accessor == rootFS) { + if (auto storePath = store->maybeParseStorePath(path.path.abs())) { + debug( + "Copying '%s' to the store again.\n" + "You can make Nix evaluate faster and copy fewer files by replacing `./.` with the `self` flake input, " + "or `builtins.path { path = ./.; name = \"source\"; }`.\n", + path); + return std::string( + fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); + } + } + return std::string(path.baseName()); +} + +StorePath EvalState::mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash) +{ + auto storePath = settings.lazyTrees + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store + std::optional _narHash; + + auto getNarHash = [&]() { + if (!_narHash) { + if (store->isValidPath(storePath)) + _narHash = store->queryPathInfo(storePath)->narHash; + else + _narHash = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()).second; + } + return _narHash; + }; + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + if (forceNarHash + || (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked(fetchSettings)) + && !input.getNarHash())) + input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); - if (originalInput.getNarHash() && narHash != *originalInput.getNarHash()) + if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) throw Error( (unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got '%s'", originalInput.to_string(), - narHash.to_string(HashFormat::SRI, true), - originalInput.getNarHash()->to_string(HashFormat::SRI, true)); + originalInput.getNarHash()->to_string(HashFormat::SRI, true), + getNarHash()->to_string(HashFormat::SRI, true)); return storePath; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3fb31fb1eb28..cf009515c57c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -18,6 +18,9 @@ #include "nix/expr/primops.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/sort.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/expr/provenance.hh" +#include "nix/util/override-provenance-source-accessor.hh" #include #include @@ -74,6 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { + waitForPath(p); if (!store->isValidPath(p)) error(p).debugThrow(); }; @@ -88,7 +92,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + // We consider virtual store paths valid here. They'll + // be devirtualized if needed elsewhere. + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -98,6 +105,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (maybePathsOut) maybePathsOut->emplace(d.drvPath); }, + [&](const NixStringContextElem::Path & p) { + // FIXME: do something? + }, }, c.raw); } @@ -239,18 +249,10 @@ void derivationToValue( auto w = state.allocValue(); w->mkAttrs(attrs); - if (!state.vImportedDrvToDerivation) { - state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); - state.eval( - state.parseExprFromString( -#include "imported-drv-to-derivation.nix.gen.hh" - , state.rootPath(CanonPath::root)), - **state.vImportedDrvToDerivation); - } + auto vImportedDrvToDerivation = state.allocValue(); + state.evalFile(state.importedDrvToDerivation, *vImportedDrvToDerivation); // has caching - state.forceFunction( - **state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); - v.mkApp(*state.vImportedDrvToDerivation, w); + v.mkApp(vImportedDrvToDerivation, w); state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } @@ -299,6 +301,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v if (!state.store->isStorePath(path2)) return std::nullopt; auto storePath = state.store->parseStorePath(path2); + state.waitForPath(storePath); if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; @@ -694,7 +697,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); @@ -781,10 +784,12 @@ struct CompareValues } }; -typedef std::list> ValueList; - static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** args, Value & v) { + /* The values are rooted via RootValue, so the lists themselves + don't need to be visible to the GC. */ + using ValueList = std::list; + state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure"); /* Get the start set. */ @@ -798,7 +803,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar ValueList workSet; for (auto elem : startSet->value->listView()) - workSet.push_back(elem); + workSet.push_back(RootValue(elem)); if (startSet->value->listSize() == 0) { v = *startSet->value; @@ -819,7 +824,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar auto cmp = CompareValues(state, noPos, ""); std::map keyToElem(cmp); while (!workSet.empty()) { - Value * e = *(workSet.begin()); + Value * e = **workSet.begin(); workSet.pop_front(); try { @@ -864,7 +869,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar } throw; } - res.push_back(e); + res.push_back(RootValue(e)); /* Call the `operator' function with `e' as argument. */ Value newElements; @@ -879,7 +884,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar for (auto elem : newElements.listView()) { state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator` // passed to builtins.genericClosure"); - workSet.push_back(elem); + workSet.push_back(RootValue(elem)); } } catch (Error & err) { err.addTrace( @@ -894,7 +899,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar /* Create the result list. */ auto list = state.buildList(res.size()); for (const auto & [n, i] : enumerate(res)) - list[n] = i; + list[n] = *i; v.mkList(list); } @@ -976,6 +981,7 @@ static RegisterPrimOp primop_break( } // Return the value we were passed. + state.forceValue(*args[0], pos); v = *args[0]; }}); @@ -1037,7 +1043,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, @@ -1148,7 +1182,7 @@ static RegisterPrimOp primop_floor({ a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat - (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. + (IEEE-754 double-precision floating-point number), an evaluation error is thrown. )", .impl = prim_floor, }); @@ -1195,7 +1229,7 @@ static RegisterPrimOp primop_tryEval({ `false` if an error was thrown) and `value`, equalling *e* if successful and `false` otherwise. `tryEval` only prevents errors created by `throw` or `assert` from being thrown. - Errors `tryEval` doesn't catch are, for example, those created + Errors that `tryEval` doesn't catch are, for example, those created by `abort` and type errors generated by builtins. Also note that this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; in (builtins.tryEval e).success` is `true`. Using @@ -1348,7 +1382,7 @@ static RegisterPrimOp primop_warn({ [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) option is set to `true` and the `--debugger` flag is given, the - interactive debugger will be started when `warn` is called (like + interactive debugger is started when `warn` is called (like [`break`](@docroot@/language/builtins.md#builtins-break)). If the @@ -1372,16 +1406,15 @@ static void prim_second(EvalState & state, const PosIdx pos, Value ** args, Valu * Derivations *************************************************************/ -static void derivationStrictInternal(EvalState & state, std::string_view name, const Bindings * attrs, Value & v); +static void derivationStrictInternal( + EvalState & state, + std::string_view name, + const Bindings * attrs, + Value & v, + std::shared_ptr provenance, + bool acceptMeta); -/* Construct (as a unobservable side effect) a Nix derivation - expression that performs the derivation described by the argument - set. Returns the original set extended with the following - attributes: `outPath' containing the primary output path of the - derivation; `drvPath' containing the path of the Nix expression; - and `type' set to `derivation' to indicate that this is a - derivation. */ -static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value ** args, Value & v) +static void prim_derivationStrictGeneric(EvalState & state, const PosIdx pos, Value ** args, Value & v, bool acceptMeta) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.derivationStrict"); @@ -1401,7 +1434,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value ** } try { - derivationStrictInternal(state, drvName, attrs, v); + derivationStrictInternal(state, drvName, attrs, v, state.evalContext.provenance, acceptMeta); } catch (Error & e) { Pos pos = state.positions[nameAttr->pos]; /* @@ -1432,6 +1465,18 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value ** } } +/* Construct a Nix derivation with metadata provenance */ +static RegisterPrimOp primop_derivationStrictWithMeta( + PrimOp{ + .name = "derivationStrictWithMeta", + .arity = 1, + .impl = + [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + prim_derivationStrictGeneric(state, pos, args, v, /*acceptMeta=*/true); + }, + .internal = true, + }); + /** * Early validation for the derivation name, for better error message. * It is checked again when constructing store paths. @@ -1455,7 +1500,13 @@ static void checkDerivationName(EvalState & state, std::string_view drvName) } } -static void derivationStrictInternal(EvalState & state, std::string_view drvName, const Bindings * attrs, Value & v) +static void derivationStrictInternal( + EvalState & state, + std::string_view drvName, + const Bindings * attrs, + Value & v, + std::shared_ptr provenance, + bool acceptMeta) { checkDerivationName(state, drvName); @@ -1580,7 +1631,19 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName the environment. */ default: - if (jsonObject) { + if (acceptMeta && i->name == EvalState::s.__meta) { + if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) { + state.forceAttrs(*i->value, pos, "while evaluating __meta"); + NixStringContext ctx; + auto obj = printValueAsJSON(state, true, *i->value, pos, ctx); + + if (!ctx.empty()) + throw Error("Derivation __meta provenance can't contain string context like store paths."); + + provenance = + std::make_shared(provenance, make_ref(obj)); + } + } else if (jsonObject) { if (i->name == state.s.structuredAttrs) continue; @@ -1704,6 +1767,10 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ + StringMap rewrites; + + std::optional drvS; + for (auto & c : context) { std::visit( overloaded{ @@ -1715,6 +1782,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName [&](const NixStringContextElem::DrvDeep & d) { /* !!! This doesn't work if readOnlyMode is set. */ StorePathSet refs; + // FIXME: don't need to wait, we only need the references. + state.waitForPath(d.drvPath); state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); @@ -1726,11 +1795,27 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName [&](const NixStringContextElem::Built & b) { drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, - [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites)); + }, + [&](const NixStringContextElem::Path & p) { + if (!drvS) + drvS = drv.unparse(*state.store, true); + if (drvS->find(p.storePath.to_string()) != drvS->npos) { + auto devirtualized = state.devirtualize(p.storePath, &rewrites); + warn( + "Using 'builtins.derivation' to create a derivation named '%s' that references the store path '%s' without a proper context. " + "The resulting derivation will not have a correct store reference, so this is unreliable and may stop working in the future.", + drvName, + state.store->printStorePath(devirtualized)); + } + }, }, c.raw); } + drv.applyRewrites(rewrites); + /* Do we have all required attributes? */ if (drv.builder == "") state.error("required attribute 'builder' missing").atPos(v).debugThrow(); @@ -1822,8 +1907,9 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName Unless we are in read-only mode, that is, in which case we do not write anything. Users commonly do this to speed up evaluation in contexts where they don't actually want to build anything. */ - auto drvPath = - settings.readOnlyMode ? computeStorePath(*state.store, drv) : state.store->writeDerivation(drv, state.repair); + auto drvPath = settings.readOnlyMode + ? computeStorePath(*state.store, drv) + : state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair, provenance); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); @@ -1850,11 +1936,21 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName v.mkAttrs(result); } +/* Construct (as a unobservable side effect) a Nix derivation + expression that performs the derivation described by the argument + set. Returns the original set extended with the following + attributes: `outPath' containing the primary output path of the + derivation; `drvPath' containing the path of the Nix expression; + and `type' set to `derivation' to indicate that this is a + derivation. */ static RegisterPrimOp primop_derivationStrict( PrimOp{ .name = "derivationStrict", .arity = 1, - .impl = prim_derivationStrict, + .impl = + [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + prim_derivationStrictGeneric(state, pos, args, v, /*acceptMeta=*/false); + }, }); /* Return a placeholder string for the specified output that will be @@ -1927,21 +2023,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ @@ -2090,14 +2186,17 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Va .debugThrow(); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { - try { - refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; - } catch (Error &) { // FIXME: should be InvalidPathError + auto storePath = state.store->toStorePath(path.path.abs()).first; + // Skip virtual paths since they don't have references and + // don't exist anyway. + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + if (auto info = state.store->maybeQueryPathInfo(state.store->toStorePath(path.path.abs()).first)) { + // Re-scan references to filter down to just the ones that actually occur in the file. + auto refsSink = PathRefScanSink::fromPaths(info->references); + refsSink << s; + refs = refsSink.getResultPaths(); + } } - // Re-scan references to filter down to just the ones that actually occur in the file. - auto refsSink = PathRefScanSink::fromPaths(refs); - refsSink << s; - refs = refsSink.getResultPaths(); } NixStringContext context; for (auto && p : std::move(refs)) { @@ -2333,6 +2432,23 @@ static RegisterPrimOp primop_hashFile({ .impl = prim_hashFile, }); +static RegisterPrimOp primop_narHash({ + .name = "__narHash", + .args = {"p"}, + .doc = R"( + Return an SRI representation of the SHA-256 hash of the NAR serialisation of the path *p*. + )", + .impl = + [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + auto path = state.realisePath(pos, *args[0]); + auto hash = + fetchToStore2(state.fetchSettings, *state.store, path.resolveSymlinks(), FetchMode::DryRun).second; + v.mkString(hash.to_string(HashFormat::SRI, true), state.mem); + }, + // FIXME: may be useful to expose to the user. + .internal = true, +}); + static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type type) { struct Constants @@ -2660,15 +2776,25 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu { NixStringContext context; auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile"); - auto contents = - state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"); + std::string contents( + state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; + StringMap rewrites; for (auto c : context) { if (auto p = std::get_if(&c.raw)) refs.insert(p->path); - else + else if (auto p = std::get_if(&c.raw)) { + if (contents.find(p->storePath.to_string()) != contents.npos) { + auto devirtualized = state.devirtualize(p->storePath, &rewrites); + warn( + "Using 'builtins.toFile' to create a file named '%s' that references the store path '%s' without a proper context. " + "The resulting file will not have a correct store reference, so this is unreliable and may stop working in the future.", + name, + state.store->printStorePath(devirtualized)); + } + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", @@ -2679,6 +2805,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu .debugThrow(); } + contents = rewriteStrings(contents, rewrites); + auto storePath = settings.readOnlyMode ? state.store->makeFixedOutputPathFromCA( name, TextInfo{ @@ -2694,7 +2822,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, - state.repair); + state.repair, + state.evalContext.provenance); }); /* Note: we don't need to add `context' to the context of the @@ -2838,11 +2967,23 @@ static void addPath( name, ContentAddressWithReferences::fromParts(method, *expectedHash, {refs})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + // FIXME: make this lazy? // FIXME: support refs in fetchToStore()? + auto path2 = path.resolveSymlinks(); + // Don't use source path provenance if we have a filter applied, since we can't accurately + // record that. Instead, use the current global provenance, since it's better than nothing. + auto path3 = filter + ? SourcePath{ + make_ref( + path2.accessor, state.evalContext.provenance), + path2.path + } + : path2; + auto dstPath = refs.empty() ? fetchToStore( state.fetchSettings, *state.store, - path.resolveSymlinks(), + path3, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, name, method, @@ -2850,7 +2991,7 @@ static void addPath( state.repair) : state.store->addToStore( name, - path.resolveSymlinks(), + path3, method, HashAlgorithm::SHA256, refs, @@ -2880,7 +3021,15 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value ** args state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath( - state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); + state, + pos, + state.computeBaseName(path, pos), + path, + args[0], + ContentAddressMethod::Raw::NixArchive, + std::nullopt, + v, + context); } static RegisterPrimOp primop_filterSource({ @@ -3050,6 +3199,8 @@ static RegisterPrimOp primop_attrNames({ Return the names of the attributes in the set *set* in an alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3082,6 +3233,8 @@ static RegisterPrimOp primop_attrValues({ .doc = R"( Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. + + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3094,7 +3247,7 @@ void prim_getAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) auto i = state.getAttr(state.symbols.create(attr), args[1]->attrs(), "in the attribute set under consideration"); // !!! add to stack trace? if (state.countCalls && i->pos) - state.attrSelects[i->pos]++; + state.attrSelects->try_emplace_or_visit(i->pos, 1, [](auto & j) { j.second++; }); state.forceValue(*i->value, pos); v = *i->value; } @@ -3107,6 +3260,8 @@ static RegisterPrimOp primop_getAttr({ aborts if the attribute doesn’t exist. This is a dynamic version of the `.` operator, since *s* is an expression rather than an identifier. + + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3195,6 +3350,8 @@ static RegisterPrimOp primop_hasAttr({ `hasAttr` returns `true` if *set* has an attribute named *s*, and `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. + + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3254,6 +3411,8 @@ static RegisterPrimOp primop_removeAttrs({ ``` evaluates to `{ y = 2; }`. + + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3341,6 +3500,8 @@ static RegisterPrimOp primop_listToAttrs({ ```nix { foo = 123; bar = 456; } ``` + + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3417,7 +3578,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3457,6 +3618,8 @@ static RegisterPrimOp primop_catAttrs({ ``` evaluates to `[1 2]`. + + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3500,6 +3663,8 @@ static RegisterPrimOp primop_functionArgs({ "Formal argument" here refers to the attributes pattern-matched by the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. + + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3532,61 +3697,147 @@ static RegisterPrimOp primop_mapAttrs({ ``` evaluates to `{ a = 10; b = 20; }`. + + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); -static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value ** args, Value & v) +static void prim_filterAttrs(EvalState & state, const PosIdx pos, Value ** args, Value & v) { - // we will first count how many values are present for each given key. - // we then allocate a single attrset and pre-populate it with lists of - // appropriate sizes, stash the pointers to the list elements of each, - // and populate the lists. after that we replace the list in the every - // attribute with the merge function application. this way we need not - // use (slightly slower) temporary storage the GC does not know about. - - struct Item - { - size_t size = 0; - size_t pos = 0; - std::optional list; - }; + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.filterAttrs"); - std::map, traceable_allocator>> attrsSeen; + if (args[1]->attrs()->empty()) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterAttrs"); + + auto attrs = state.buildBindings(args[1]->attrs()->size()); + + for (auto & i : *args[1]->attrs()) { + Value * vName = Value::toPtr(state.symbols[i.name]); + Value * callArgs[] = {vName, i.value}; + Value res; + state.callFunction(*args[0], callArgs, res, noPos); + if (state.forceBool( + res, pos, "while evaluating the return value of the filtering function passed to builtins.filterAttrs")) + attrs.insert(i.name, i.value); + } + + v.mkAttrs(attrs.alreadySorted()); +} + +static RegisterPrimOp primop_filterAttrs({ + .name = "__filterAttrs", + .args = {"f", "attrset"}, + .doc = R"( + Return an attribute set consisting of the attributes in *attrset* for which + the function *f* returns `true`. The function *f* is called with two arguments: + the name of the attribute and the value of the attribute. For example, + + ```nix + builtins.filterAttrs (name: value: name == "foo") { foo = 1; bar = 2; } + ``` + evaluates to `{ foo = 1; }`. + )", + .impl = prim_filterAttrs, +}); + +/** + * A (name, value) pair collected by primops that group values by name + * (`zipAttrsWith`, `groupBy`). + * + * Note that vectors of these can be invisible to the GC (provided that + * the values are reachable in some other way for the duration of the + * primop, e.g. from the primop's arguments), which avoids allocating + * GC-visible (uncollectable) storage for temporaries. The latter is + * expensive and a source of GC allocation lock contention during + * parallel evaluation. + */ +struct NameValue +{ + Symbol name; + Value * value; +}; + +/** + * Stably sort `items` by name, so that the values of equal names form + * contiguous runs in their original relative order, and return the + * number of distinct names. The resulting order is the same as that of + * a std::map. + */ +static size_t sortByName(std::vector & items) +{ + std::stable_sort( + items.begin(), items.end(), [](const NameValue & a, const NameValue & b) { return a.name < b.name; }); + + size_t nrNames = 0; + for (size_t i = 0; i < items.size(); ++i) + if (i == 0 || items[i - 1].name != items[i].name) + nrNames++; + return nrNames; +} + +/** + * Call `f(name, list)` for every distinct name in `items` (which must + * have been sorted with `sortByName()`), where `list` is a ListBuilder + * containing the values associated with that name. + */ +template +static void forEachByName(EvalState & state, const std::vector & items, F f) +{ + for (size_t i = 0; i < items.size();) { + auto sym = items[i].name; + size_t j = i; + while (j < items.size() && items[j].name == sym) + j++; + auto list = state.buildList(j - i); + for (size_t k = i; k < j; ++k) + list[k - i] = items[k].value; + f(sym, list); + i = j; + } +} + +static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); const auto listItems = args[1]->listView(); + size_t nrAttrs = 0; for (auto & vElem : listItems) { state.forceAttrs( *vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"); - for (auto & attr : *vElem->attrs()) - attrsSeen.try_emplace(attr.name).first->second.size++; + nrAttrs += vElem->attrs()->size(); } - for (auto & [sym, elem] : attrsSeen) - elem.list.emplace(state.buildList(elem.size)); + /* The vector is invisible to the GC, but that's fine: the values are + kept alive by the attrsets in *args[1]. */ + std::vector attrsSeen; + attrsSeen.reserve(nrAttrs); - for (auto & vElem : listItems) { - for (auto & attr : *vElem->attrs()) { - auto & item = attrsSeen.at(attr.name); - (*item.list)[item.pos++] = attr.value; - } - } + for (auto & vElem : listItems) + for (auto & attr : *vElem->attrs()) + attrsSeen.push_back({attr.name, attr.value}); - auto attrs = state.buildBindings(attrsSeen.size()); + auto attrs = state.buildBindings(sortByName(attrsSeen)); - for (auto & [sym, elem] : attrsSeen) { + forEachByName(state, attrsSeen, [&](Symbol sym, ListBuilder & list) { auto name = Value::toPtr(state.symbols[sym]); auto call1 = state.allocValue(); call1->mkApp(args[0], name); auto call2 = state.allocValue(); auto arg = state.allocValue(); - arg->mkList(*elem.list); + arg->mkList(list); call2->mkApp(call1, arg); attrs.insert(sym, call2); - } + }); v.mkAttrs(attrs.alreadySorted()); } @@ -3619,6 +3870,8 @@ static RegisterPrimOp primop_zipAttrsWith({ b = { name = "b"; values = [ "z" ]; }; } ``` + + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3653,8 +3906,9 @@ static void prim_elemAt(EvalState & state, const PosIdx pos, Value ** args, Valu state.error("'builtins.elemAt' called with index %d on a list of size %d", n, args[0]->listSize()) .atPos(pos) .debugThrow(); - state.forceValue(*args[0]->listView()[n], pos); - v = *args[0]->listView()[n]; + auto ptr = args[0]->listView()[n]; + state.forceValue(*ptr, pos); + v = *ptr; } static RegisterPrimOp primop_elemAt({ @@ -3684,6 +3938,8 @@ static RegisterPrimOp primop_head({ Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. + + Has constant time complexity. )", .impl = prim_head, }); @@ -3749,6 +4005,10 @@ static RegisterPrimOp primop_map({ ``` evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -3798,6 +4058,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -3821,6 +4082,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -3830,12 +4092,7 @@ static void prim_concatLists(EvalState & state, const PosIdx pos, Value ** args, { state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists"); auto listView = args[0]->listView(); - state.concatLists( - v, - args[0]->listSize(), - listView.data(), - pos, - "while evaluating a value of the list passed to builtins.concatLists"); + state.concatLists(v, listView.span(), pos, "while evaluating a value of the list passed to builtins.concatLists"); } static RegisterPrimOp primop_concatLists({ @@ -3891,17 +4148,40 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. + e.g. + ```nix + foldl' op nul [ x0 x1 x2 ] + = + let + strictly = f: a: builtins.seq a (f a); + y0 = op nul x0; + y1 = strictly op y0 x1; + y2 = strictly op y1 x2; + in + y2 + + # and, ignoring strictness/laziness + == + op (op (op nul x0) x1) x2 + ``` For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. The first argument of `op` is the accumulator whereas the second - argument is the current element being processed. The return value - of each application of `op` is evaluated immediately, even for - intermediate values. + argument is the current element being processed. + + The return value of each application of `op` is evaluated immediately, + even for intermediate values. + This way, `foldl'` can operate in constant stack space, allowing it to operate on large lists, + regardless of [max-call-depth](@docroot@/command-ref/conf-file.md#conf-max-call-depth). + + Conventionally, a fold function without the `'` ("prime") preserves laziness, + but lacks these benefits. + See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). + + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -3916,8 +4196,8 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value ** arg std::string_view errorCtx = any ? "while evaluating the return value of the function passed to builtins.any" : "while evaluating the return value of the function passed to builtins.all"; - Value vTmp; for (auto elem : args[1]->listView()) { + Value vTmp; state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { @@ -3940,6 +4220,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -3955,6 +4236,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -3993,6 +4275,8 @@ static RegisterPrimOp primop_genList({ ``` returns the list `[ 0 1 4 9 16 ]`. + + Has linear time complexity. )", .impl = prim_genList, }); @@ -4103,6 +4387,9 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. + + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4164,6 +4451,8 @@ static RegisterPrimOp primop_partition({ ```nix { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` + + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4173,26 +4462,24 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value ** args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.groupBy"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.groupBy"); - ValueVectorMap attrs; + const auto listItems = args[1]->listView(); - for (auto vElem : args[1]->listView()) { + /* The vector is invisible to the GC, but that's fine: the values are + kept alive by the list in *args[1]. */ + std::vector items; + items.reserve(listItems.size()); + + for (auto vElem : listItems) { Value res; state.callFunction(*args[0], *vElem, res, pos); auto name = state.forceStringNoCtx( res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); - auto sym = state.symbols.create(name); - auto vector = attrs.try_emplace(sym, {}).first; - vector->second.push_back(vElem); + items.push_back({state.symbols.create(name), vElem}); } - auto attrs2 = state.buildBindings(attrs.size()); + auto attrs2 = state.buildBindings(sortByName(items)); - for (auto & i : attrs) { - auto size = i.second.size(); - auto list = state.buildList(size); - memcpy(list.elems, i.second.data(), sizeof(Value *) * size); - attrs2.alloc(i.first).mkList(list); - } + forEachByName(state, items, [&](Symbol sym, ListBuilder & list) { attrs2.alloc(sym).mkList(list); }); v.mkAttrs(attrs2.alreadySorted()); } @@ -4217,6 +4504,8 @@ static RegisterPrimOp primop_groupBy({ ```nix { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` + + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -4599,8 +4888,9 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args, state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); NixStringContext context; // discarded - auto s = - state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); + auto s = state.devirtualize( + state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"), + context); v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem); } @@ -5046,6 +5336,8 @@ static RegisterPrimOp primop_replaceStrings({ ``` evaluates to `"fabir"`. + + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); @@ -5229,9 +5521,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) )", }); - if (!settings.pureEval) { - v.mkInt(time(nullptr)); - } + v.mkInt(time(nullptr)); addConstant( "__currentTime", v, @@ -5259,8 +5549,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) .impureOnly = true, }); - if (!settings.pureEval) - v.mkString(settings.getCurrentSystem(), mem); + v.mkString(settings.getCurrentSystem(), mem); addConstant( "__currentSystem", v, @@ -5446,6 +5735,16 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) .type = nFunction, }); + auto vDerivationWithMeta = allocValue(); + if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) { + addConstant( + "derivationWithMeta", + vDerivationWithMeta, + { + .type = nFunction, + }); + } + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ const_cast(getBuiltins().attrs())->sort(); @@ -5454,7 +5753,14 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - evalFile(derivationInternal, *vDerivation); + auto vDerivationValue = allocValue(); + evalFile(derivationInternal, *vDerivationValue); + + callFunction(*vDerivationValue, getBuiltin("derivationStrict"), *vDerivation, PosIdx()); + + if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) + callFunction( + *vDerivationValue, ***get(internalPrimOps, "derivationStrictWithMeta"), *vDerivationWithMeta, PosIdx()); } } // namespace nix diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index d5d5de0b9aaf..569e8a924d23 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -8,10 +8,16 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value ** args, Value & v) { - NixStringContext context; + NixStringContext context, filtered; + auto s = state.coerceToString( pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); - v.mkString(*s, state.mem); + + for (auto & c : context) + if (auto * p = std::get_if(&c.raw)) + filtered.insert(*p); + + v.mkString(*s, filtered, state.mem); } static RegisterPrimOp primop_unsafeDiscardStringContext({ @@ -23,11 +29,19 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({ .impl = prim_unsafeDiscardStringContext, }); +bool hasContext(const NixStringContext & context) +{ + for (auto & c : context) + if (!std::get_if(&c.raw)) + return true; + return false; +} + static void prim_hasContext(EvalState & state, const PosIdx pos, Value ** args, Value & v) { NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); - v.mkBool(!context.empty()); + v.mkBool(hasContext(context)); } static RegisterPrimOp primop_hasContext( @@ -62,6 +76,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p NixStringContext context2; for (auto && c : context) { if (auto * ptr = std::get_if(&c.raw)) { + state.waitForPath(ptr->drvPath); // FIXME: why? context2.emplace(NixStringContextElem::Opaque{.path = ptr->drvPath}); } else { /* Can reuse original item */ @@ -133,6 +148,11 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V above does not make much sense. */ return std::move(c); }, + [&](const NixStringContextElem::Path & p) -> NixStringContextElem::DrvDeep { + state.error("`addDrvOutputDependencies` does not work on a string without context") + .atPos(pos) + .debugThrow(); + }, }, context.begin()->raw)}), }; @@ -201,6 +221,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args, contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; }, + [&](NixStringContextElem::Path && p) {}, }, ((NixStringContextElem &&) i).raw); } diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index dbb8c2186889..06c7eb8a67e0 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -26,6 +26,7 @@ Note that `derivation` is very bare-bones, and provides almost no commands during the build. Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. */ +drvFunc: drvAttrs@{ outputs ? [ "out" ], ... @@ -33,7 +34,7 @@ drvAttrs@{ let - strict = derivationStrict drvAttrs; + strict = drvFunc drvAttrs; commonAttrs = drvAttrs @@ -46,7 +47,7 @@ let outputToAttrListElement = outputName: { name = outputName; value = commonAttrs // { - outPath = builtins.getAttr outputName strict; + outPath = strict.${outputName}; drvPath = strict.drvPath; type = "derivation"; inherit outputName; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 2e01d67172d3..aebafb35b0d4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -2,7 +2,6 @@ #include "nix/store/store-open.hh" #include "nix/store/realisation.hh" #include "nix/store/make-content-addressed.hh" -#include "nix/util/url.hh" #include "nix/util/environment-variables.hh" namespace nix { @@ -24,6 +23,8 @@ static void runFetchClosureWithRewrite( const std::optional & toPathMaybe, Value & v) { + if (toPathMaybe) + state.store->addTempRoot(*toPathMaybe); // establish toPath or throw @@ -75,6 +76,7 @@ static void runFetchClosureWithRewrite( static void runFetchClosureWithContentAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -104,6 +106,7 @@ static void runFetchClosureWithContentAddressedPath( static void runFetchClosureWithInputAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -136,7 +139,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs()) { - const auto & attrName = state.symbols[attr.name]; + std::string_view attrName = state.symbols[attr.name]; auto attrHint = [&]() -> std::string { return fmt("while evaluating the attribute '%s' passed to builtins.fetchClosure", attrName); }; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9347645cc204..172c37948c93 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,9 +1,7 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" -#include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/url.hh" #include "nix/util/url-parts.hh" namespace nix { @@ -81,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); - auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store); + auto [storePath, accessor, input2] = input.fetchToStore(state.fetchSettings, *state.store); auto attrs2 = state.buildBindings(8); state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6e06bdbe973f..392294abc7e0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,7 +1,9 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" @@ -16,10 +18,104 @@ #include #include -#include namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +void LazyFetcherAttr::anchor() {} + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -30,12 +126,16 @@ void emitTreeAttrs( { auto attrs = state.buildBindings(100); - state.mkStorePathString(storePath, attrs.alloc(state.s.outPath)); + auto & vStorePath = attrs.alloc(state.s.outPath); + state.mkStorePathString(storePath, vStorePath); // FIXME: support arbitrary input attributes. if (auto narHash = input.getNarHash()) attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem); + else + // Lazily compute the NAR hash for backward compatibility. + attrs.alloc("narHash").mkApp(**get(state.internalPrimOps, "narHash"), &vStorePath); if (input.getType() == "git") attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); @@ -52,7 +152,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); @@ -77,7 +179,6 @@ struct FetchTreeParams bool emptyRevFallback = false; bool allowNameArgument = false; bool isFetchGit = false; - bool isFinal = false; }; static void fetchTree( @@ -151,11 +252,6 @@ static void fetchTree( attrs.emplace("exportIgnore", Explicit{true}); } - // fetchTree should fetch git repos with shallow = true by default - if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) { - attrs.emplace("shallow", Explicit{true}); - } - if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) state.error("argument 'name' isn’t supported in call to '%s'", fetcher) @@ -184,17 +280,11 @@ static void fetchTree( } input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - state - .error( - "passing a string argument to '%s' requires the 'flakes' experimental feature", fetcher) - .atPos(pos) - .debugThrow(); input = fetchers::Input::fromURL(state.fetchSettings, url); } } - if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect()) input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first; if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) { @@ -213,17 +303,13 @@ static void fetchTree( state.checkURI(input.toURLString()); - if (params.isFinal) { + if (input.getNarHash()) input.attrs.insert_or_assign("__final", Explicit(true)); - } else { - if (input.isFinal()) - throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); - } auto cachedInput = state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); - auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true); emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false); } @@ -318,7 +404,6 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -358,19 +443,6 @@ static RegisterPrimOp primop_fetchTree({ return doc; }(), .impl = prim_fetchTree, - .experimentalFeature = Xp::FetchTree, -}); - -void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Value & v) -{ - fetchTree(state, pos, args, v, {.isFinal = true}); -} - -static RegisterPrimOp primop_fetchFinalTree({ - .name = "fetchFinalTree", - .args = {"input"}, - .impl = prim_fetchFinalTree, - .internal = true, }); static void fetch( @@ -477,35 +549,39 @@ static void fetch( } } - // Download the file/tarball if substitution failed or no hash was provided - auto storePath = unpack ? fetchToStore( - state.fetchSettings, - *state.store, - fetchers::downloadTarball(*state.store, state.fetchSettings, *url), - FetchMode::Copy, - name) - : fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; - - if (expectedHash) { - auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashPath( - {state.store->requireStoreObjectAccessor(storePath)}, - FileSerialisationMethod::Flat, - HashAlgorithm::SHA256) - .hash; - if (hash != *expectedHash) { - state - .error( - "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, - expectedHash->to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)) - .withExitStatus(102) - .debugThrow(); + if (unpack) { + auto attrs = fetchers::Attrs{ + {"type", "tarball"}, + {"url", *url}, + {"name", name}, + }; + if (expectedHash) + attrs.emplace("narHash", expectedHash->to_string(HashFormat::SRI, true)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); + auto cachedInput = + state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, false); + state.mkStorePathString(storePath, v); + } else { + auto storePath = fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; + if (expectedHash) { + auto hash = hashPath( + {state.store->requireStoreObjectAccessor(storePath)}, + FileSerialisationMethod::Flat, + HashAlgorithm::SHA256) + .hash; + if (hash != *expectedHash) + state + .error( + "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, + expectedHash->to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)) + .withExitStatus(102) + .debugThrow(); } + state.allowAndSetStorePathString(storePath, v); } - - state.allowAndSetStorePathString(storePath, v); } static void prim_fetchurl(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -600,7 +676,12 @@ static RegisterPrimOp primop_fetchGit({ - `url` - The URL of the repo. + The [Git URL] of the repo. SCP-like syntax is supported, but relative + paths are rewritten to absolute ones. For example: + + `git@github.com:repo/path` becomes `ssh://git@github.com/repo/path` + + [Git URL]: https://git-scm.com/docs/git-clone#_git_urls - `name` (default: `source`) @@ -719,7 +800,7 @@ static RegisterPrimOp primop_fetchGit({ name in the `ref` attribute. However, if the revision you're looking for is in a future - branch for the non-default branch you will need to specify the + branch for the non-default branch you need to specify the the `ref` attribute as well. ```nix diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index b8abc6409af9..d62b6df4ea20 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -10,3 +10,7 @@ sources += files( 'fetchTree.cc', 'fromTOML.cc', ) + +if wasmtime_required.enabled() + sources += files('wasm.cc') +endif diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc new file mode 100644 index 000000000000..b59cbec93e90 --- /dev/null +++ b/src/libexpr/primops/wasm.cc @@ -0,0 +1,741 @@ +#include "nix/expr/primops.hh" +#include "nix/expr/eval-inline.hh" + +#include +#include + +using namespace wasmtime; + +namespace nix { + +using ValueId = uint32_t; + +template +T unwrap(Result && res) +{ + if (res) + return res.ok(); + throw Error(res.err().message()); +} + +static Engine & getEngine() +{ + static Engine engine = []() { + wasmtime::Config config; + config.pooling_allocation_strategy(PoolAllocationConfig()); + config.memory_init_cow(true); + return Engine(std::move(config)); + }(); + return engine; +} + +static std::span string2span(std::string_view s) +{ + return std::span((uint8_t *) s.data(), s.size()); +} + +static std::string_view span2string(std::span s) +{ + return std::string_view((char *) s.data(), s.size()); +} + +template +static std::span subspan(std::span s, size_t len) +{ + if (s.size() < len * sizeof(T)) + throw Error("Wasm memory access out of bounds"); + return std::span((T *) s.data(), len); +} + +// FIXME: move to wasmtime C++ wrapper. +class InstancePre +{ + WASMTIME_OWN_WRAPPER(InstancePre, wasmtime_instance_pre); + +public: + TrapResult instantiate(wasmtime::Store::Context cx) + { + wasmtime_instance_t instance; + wasm_trap_t * trap = nullptr; + auto * error = wasmtime_instance_pre_instantiate(ptr.get(), cx.capi(), &instance, &trap); + if (error != nullptr) { + return TrapError(wasmtime::Error(error)); + } + if (trap != nullptr) { + return TrapError(Trap(trap)); + } + return Instance(instance); + } +}; + +TrapResult instantiate_pre(Linker & linker, const Module & m) +{ + wasmtime_instance_pre_t * instance_pre; + auto * error = wasmtime_linker_instantiate_pre(linker.capi(), m.capi(), &instance_pre); + if (error != nullptr) { + return TrapError(wasmtime::Error(error)); + } + return InstancePre(instance_pre); +} + +static void regFuns(Linker & linker, bool useWasi); + +struct NixWasmInstancePre +{ + Engine & engine = getEngine(); + std::string name; + bool useWasi = false; + InstancePre instancePre; + + InstancePre compile(std::span bytes) + { + // Compile the module + auto module = unwrap(Module::compile(engine, bytes)); + + // Auto-detect WASI by checking for wasi_snapshot_preview1 imports. + for (const auto & ref : module.imports()) + if (const_cast &>(ref).module() == "wasi_snapshot_preview1") { + useWasi = true; + break; + } + + // Create linker with appropriate WASI support + Linker linker(engine); + if (useWasi) + unwrap(linker.define_wasi()); + regFuns(linker, useWasi); + + return unwrap(instantiate_pre(linker, module)); + } + + NixWasmInstancePre(SourcePath wasmPath) + : name(wasmPath.baseName()) + , instancePre(compile(string2span(wasmPath.readFile()))) + { + } + + NixWasmInstancePre(std::string_view wat) + : name("") + , instancePre([&] { + auto wasm = unwrap(wat2wasm(wat)); + return compile(std::span(wasm)); + }()) + { + } +}; + +struct NixWasmInstance +{ + EvalState & state; + ref pre; + wasmtime::Store wasmStore; + wasmtime::Store::Context wasmCtx; + Instance instance; + Memory memory_; + + ValueVector values; + std::exception_ptr ex; + + std::optional functionName; + + ValueId resultId = 0; + + std::string logPrefix; + + NixWasmInstance(EvalState & _state, ref _pre) + : state(_state) + , pre(_pre) + , wasmStore(pre->engine) + , wasmCtx(wasmStore) + , instance(unwrap(pre->instancePre.instantiate(wasmCtx))) + , memory_(getExport("memory")) + , logPrefix(pre->name) + { + wasmCtx.set_data(this); + + /* Reserve value ID 0 so it can be used in functions like get_attr() to denote a missing attribute. */ + values.push_back(nullptr); + } + + ValueId addValue(Value * v) + { + auto id = values.size(); + values.emplace_back(v); + return id; + } + + std::pair allocValue() + { + auto v = state.allocValue(); + auto id = addValue(v); + return {id, *v}; + } + + Value & getValue(ValueId id) + { + if (id >= values.size() || id == 0) + throw Error("invalid ValueId %d", id); + return *values[id]; + } + + template + T getExport(std::string_view name) + { + auto ext = instance.get(wasmCtx, name); + if (!ext) + throw Error("Wasm module '%s' does not export '%s'", pre->name, name); + auto res = std::get_if(&*ext); + if (!res) + throw Error("export '%s' of Wasm module '%s' does not have the right type", name, pre->name); + return *res; + } + + std::vector runFunction(std::string_view name, const std::vector & args) + { + functionName = name; + return unwrap(getExport(name).call(wasmCtx, args)); + } + + auto memory() + { + return memory_.data(wasmCtx); + } + + std::monostate panic(uint32_t ptr, uint32_t len) + { + throw Error("Wasm panic: %s", Uncolored(span2string(memory().subspan(ptr, len)))); + } + + std::monostate warn(uint32_t ptr, uint32_t len) + { + doWarn(span2string(memory().subspan(ptr, len))); + return {}; + } + + void doWarn(std::string_view s) + { + if (functionName) + nix::warn("'%s' function '%s': %s", logPrefix, functionName.value_or(""), s); + else + nix::warn("'%s': %s", logPrefix, s); + } + + uint32_t get_type(ValueId valueId) + { + auto & value = getValue(valueId); + state.forceValue(value, noPos); + auto t = value.type(); + return t == nInt ? 1 + : t == nFloat ? 2 + : t == nBool ? 3 + : t == nString ? 4 + : t == nPath ? 5 + : t == nNull ? 6 + : t == nAttrs ? 7 + : t == nList ? 8 + : t == nFunction ? 9 + : []() -> int { throw Error("unsupported type"); }(); + } + + ValueId make_int(int64_t n) + { + auto [valueId, value] = allocValue(); + value.mkInt(n); + return valueId; + } + + int64_t get_int(ValueId valueId) + { + return state.forceInt(getValue(valueId), noPos, "while evaluating a value from Wasm").value; + } + + ValueId make_float(double x) + { + auto [valueId, value] = allocValue(); + value.mkFloat(x); + return valueId; + } + + double get_float(ValueId valueId) + { + return state.forceFloat(getValue(valueId), noPos, "while evaluating a value from Wasm"); + } + + ValueId make_string(uint32_t ptr, uint32_t len) + { + auto [valueId, value] = allocValue(); + value.mkString(span2string(memory().subspan(ptr, len)), state.mem); + return valueId; + } + + uint32_t copy_string(ValueId valueId, uint32_t ptr, uint32_t maxLen) + { + auto s = state.forceString(getValue(valueId), noPos, "while evaluating a value from Wasm"); + if (s.size() <= maxLen) { + auto buf = memory().subspan(ptr, maxLen); + memcpy(buf.data(), s.data(), s.size()); + } + return s.size(); + } + + ValueId make_path(ValueId baseId, uint32_t ptr, uint32_t len) + { + auto & baseValue = getValue(baseId); + state.forceValue(baseValue, noPos); + if (baseValue.type() != nPath) + throw Error("make_path expects a path value"); + auto base = baseValue.path(); + + auto [valueId, value] = allocValue(); + value.mkPath({base.accessor, CanonPath(span2string(memory().subspan(ptr, len)), base.path)}, state.mem); + return valueId; + } + + uint32_t copy_path(ValueId valueId, uint32_t ptr, uint32_t maxLen) + { + auto & v = getValue(valueId); + state.forceValue(v, noPos); + if (v.type() != nPath) + throw Error("copy_path expects a path value"); + auto path = v.path().path; + auto s = path.abs(); + if (s.size() <= maxLen) { + auto buf = memory().subspan(ptr, maxLen); + memcpy(buf.data(), s.data(), s.size()); + } + return s.size(); + } + + ValueId make_bool(int32_t b) + { + return addValue(state.getBool(b)); + } + + int32_t get_bool(ValueId valueId) + { + return state.forceBool(getValue(valueId), noPos, "while evaluating a value from Wasm"); + } + + ValueId make_null() + { + return addValue(&Value::vNull); + } + + ValueId make_list(uint32_t ptr, uint32_t len) + { + auto vs = subspan(memory().subspan(ptr), len); + + auto [valueId, value] = allocValue(); + + auto list = state.buildList(len); + for (const auto & [n, v] : enumerate(list)) + v = &getValue(vs[n]); // FIXME: endianness + value.mkList(list); + + return valueId; + } + + uint32_t copy_list(ValueId valueId, uint32_t ptr, uint32_t maxLen) + { + auto & value = getValue(valueId); + state.forceList(value, noPos, "while getting a list from Wasm"); + + if (value.listSize() <= maxLen) { + auto out = subspan(memory().subspan(ptr), value.listSize()); + + for (const auto & [n, elem] : enumerate(value.listView())) + out[n] = addValue(elem); + } + + return value.listSize(); + } + + ValueId make_attrset(uint32_t ptr, uint32_t len) + { + auto mem = memory(); + + struct Attr + { + // FIXME: endianness + uint32_t attrNamePtr; + uint32_t attrNameLen; + ValueId value; + }; + + auto attrs = subspan(mem.subspan(ptr), len); + + auto [valueId, value] = allocValue(); + auto builder = state.buildBindings(len); + for (auto & attr : attrs) + builder.insert( + state.symbols.create(span2string(mem.subspan(attr.attrNamePtr, attr.attrNameLen))), + &getValue(attr.value)); + value.mkAttrs(builder); + + return valueId; + } + + uint32_t copy_attrset(ValueId valueId, uint32_t ptr, uint32_t maxLen) + { + auto & value = getValue(valueId); + state.forceAttrs(value, noPos, "while copying an attrset into Wasm"); + + if (value.attrs()->size() <= maxLen) { + // FIXME: endianness. + struct Attr + { + ValueId value; + uint32_t nameLen; + }; + + auto buf = subspan(memory().subspan(ptr), maxLen); + + // FIXME: for determinism, we should return attributes in lexicographically sorted order. + for (const auto & [n, attr] : enumerate(*value.attrs())) { + buf[n].value = addValue(attr.value); + buf[n].nameLen = state.symbols[attr.name].size(); + } + } + + return value.attrs()->size(); + } + + std::monostate copy_attrname(ValueId valueId, uint32_t attrIdx, uint32_t ptr, uint32_t len) + { + auto & value = getValue(valueId); + state.forceAttrs(value, noPos, "while copying an attr name into Wasm"); + + auto & attrs = *value.attrs(); + + if ((size_t) attrIdx >= attrs.size()) + throw Error("copy_attrname: attribute index out of bounds"); + + std::string_view name = state.symbols[attrs[attrIdx].name]; + + if ((size_t) len != name.size()) + throw Error("copy_attrname: buffer length does not match attribute name length"); + + memcpy(memory().subspan(ptr, len).data(), name.data(), name.size()); + + return {}; + } + + ValueId get_attr(ValueId valueId, uint32_t ptr, uint32_t len) + { + auto attrName = span2string(memory().subspan(ptr, len)); + + auto & value = getValue(valueId); + state.forceAttrs(value, noPos, "while getting an attribute from Wasm"); + + auto attr = value.attrs()->get(state.symbols.create(attrName)); + + return attr ? addValue(attr->value) : 0; + } + + ValueId call_function(ValueId funId, uint32_t ptr, uint32_t len) + { + auto & fun = getValue(funId); + state.forceFunction(fun, noPos, "while calling a function from Wasm"); + + ValueVector args; + for (auto argId : subspan(memory().subspan(ptr), len)) + args.push_back(&getValue(argId)); + + auto [valueId, value] = allocValue(); + + state.callFunction(fun, args, value, noPos); + + return valueId; + } + + ValueId make_app(ValueId funId, uint32_t ptr, uint32_t len) + { + if (!len) + return funId; + + auto args = subspan(memory().subspan(ptr), len); + + auto res = &getValue(funId); + + while (!args.empty()) { + auto arg = &getValue(args[0]); + auto tmp = state.allocValue(); + tmp->mkApp(res, arg); + res = tmp; + args = args.subspan(1); + } + + return addValue(res); + } + + /** + * Read the contents of a file into Wasm memory. This is like calling `builtins.readFile`, except that it can handle + * binary files that cannot be represented as Nix strings. + */ + uint32_t read_file(ValueId pathId, uint32_t ptr, uint32_t len) + { + auto & pathValue = getValue(pathId); + auto path = state.realisePath(noPos, pathValue); + + auto contents = path.readFile(); + + if (contents.size() > std::numeric_limits::max()) + throw Error("file '%s' is too large to process in Wasm (size: %d)", path, contents.size()); + + // FIXME: this is an inefficient interface since it may cause the file to be read twice. + if (contents.size() <= len) { + auto buf = memory().subspan(ptr, len); + memcpy(buf.data(), contents.data(), contents.size()); + } + + return contents.size(); + } +}; + +template +static void regFun(Linker & linker, std::string_view name, R (NixWasmInstance::*f)(Args...)) +{ + unwrap(linker.func_wrap("env", name, [f](Caller caller, Args... args) -> Result { + try { + auto instance = std::any_cast(caller.context().get_data()); + return (*instance.*f)(args...); + } catch (std::exception & e) { + return Trap(e.what()); + } catch (...) { + return Trap("unknown exception"); + } + })); +} + +static void regFuns(Linker & linker, bool useWasi) +{ + regFun(linker, "panic", &NixWasmInstance::panic); + regFun(linker, "warn", &NixWasmInstance::warn); + regFun(linker, "get_type", &NixWasmInstance::get_type); + regFun(linker, "make_int", &NixWasmInstance::make_int); + regFun(linker, "get_int", &NixWasmInstance::get_int); + regFun(linker, "make_float", &NixWasmInstance::make_float); + regFun(linker, "get_float", &NixWasmInstance::get_float); + regFun(linker, "make_string", &NixWasmInstance::make_string); + regFun(linker, "copy_string", &NixWasmInstance::copy_string); + regFun(linker, "make_path", &NixWasmInstance::make_path); + regFun(linker, "copy_path", &NixWasmInstance::copy_path); + regFun(linker, "make_bool", &NixWasmInstance::make_bool); + regFun(linker, "get_bool", &NixWasmInstance::get_bool); + regFun(linker, "make_null", &NixWasmInstance::make_null); + regFun(linker, "make_list", &NixWasmInstance::make_list); + regFun(linker, "copy_list", &NixWasmInstance::copy_list); + regFun(linker, "make_attrset", &NixWasmInstance::make_attrset); + regFun(linker, "copy_attrset", &NixWasmInstance::copy_attrset); + regFun(linker, "copy_attrname", &NixWasmInstance::copy_attrname); + regFun(linker, "get_attr", &NixWasmInstance::get_attr); + regFun(linker, "call_function", &NixWasmInstance::call_function); + regFun(linker, "make_app", &NixWasmInstance::make_app); + regFun(linker, "read_file", &NixWasmInstance::read_file); + + if (useWasi) { + unwrap(linker.func_wrap( + "env", "return_to_nix", [](Caller caller, ValueId resultId) -> Result { + auto instance = std::any_cast(caller.context().get_data()); + instance->resultId = resultId; + return Trap("return_to_nix"); + })); + } +} + +template +struct LazyMakeRef +{ + ref p; + + template + LazyMakeRef(Args &&... args) + : p(make_ref(std::move(args...))) + { + } +}; + +static NixWasmInstance instantiateWasm(EvalState & state, const SourcePath & wasmPath) +{ + // FIXME: make this a weak Boehm GC pointer so that it can be freed during GC. + // FIXME: move to EvalState? + // Note: InstancePre in Rust is Send+Sync so it should be safe to share between threads. + static boost::concurrent_flat_map> instancesPre; + + std::shared_ptr instancePre; + + instancesPre.try_emplace_and_cvisit( + wasmPath, wasmPath, [&](auto & i) { instancePre = i.second.p; }, [&](auto & i) { instancePre = i.second.p; }); + + return NixWasmInstance{state, ref(instancePre)}; +} + +/** + * Callback for WASI stdout/stderr writes. It splits the output into lines and logs each line separately. + */ +struct WasiLogger +{ + NixWasmInstance & instance; + + std::string data; + + ~WasiLogger() + { + if (!data.empty()) + instance.doWarn(data); + } + + void operator()(std::string_view s) + { + data.append(s); + + while (true) { + auto pos = data.find('\n'); + if (pos == std::string_view::npos) + break; + instance.doWarn(data.substr(0, pos)); + data.erase(0, pos + 1); + } + } +}; + +static void prim_wasm(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument to `builtins.wasm`"); + + // Check for unknown attributes + for (auto & attr : *args[0]->attrs()) { + auto name = state.symbols[attr.name]; + if (name != "path" && name != "wat" && name != "function") + throw Error("unknown attribute '%s' in first argument to `builtins.wasm`", name); + } + + auto pathAttr = args[0]->attrs()->get(state.symbols.create("path")); + auto watAttr = args[0]->attrs()->get(state.symbols.create("wat")); + + if (pathAttr && watAttr) + throw Error("'path' and 'wat' are mutually exclusive in first argument to `builtins.wasm`"); + if (!pathAttr && !watAttr) + throw Error("missing required 'path' or 'wat' attribute in first argument to `builtins.wasm`"); + + // Second argument is the value to pass to the function + auto argValue = args[1]; + + try { + auto instance = pathAttr ? instantiateWasm(state, state.realisePath(pos, *pathAttr->value)) + : NixWasmInstance{ + state, + make_ref(state.forceStringNoCtx( + *watAttr->value, pos, "while evaluating the 'wat' attribute"))}; + + // Extract 'function' attribute (optional for wasi, required for non-wasi) + std::string functionName; + auto functionAttr = args[0]->attrs()->get(state.symbols.create("function")); + if (instance.pre->useWasi) { + functionName = "_start"; + if (functionAttr) + throw Error("'function' attribute is not allowed for WASI modules"); + } else { + if (!functionAttr) + throw Error( + "missing required 'function' attribute in first argument to `builtins.wasm` for non-WASI modules"); + functionName = std::string( + state.forceStringNoCtx(*functionAttr->value, pos, "while evaluating the 'function' attribute")); + } + + debug("calling wasm module"); + + auto argId = instance.addValue(argValue); + + if (instance.pre->useWasi) { + WasiLogger logger{instance}; + + auto loggerTrampoline = [](void * data, const unsigned char * buf, size_t len) -> ptrdiff_t { + auto logger = static_cast(data); + (*logger)(std::string_view((const char *) buf, len)); + return len; + }; + + WasiConfig wasiConfig; + wasi_config_set_stdout_custom(wasiConfig.capi(), loggerTrampoline, &logger, nullptr); + wasi_config_set_stderr_custom(wasiConfig.capi(), loggerTrampoline, &logger, nullptr); + wasiConfig.argv({"wasi", std::to_string(argId)}); + unwrap(instance.wasmStore.context().set_wasi(std::move(wasiConfig))); + + auto res = instance.getExport(functionName).call(instance.wasmCtx, {}); + if (!instance.resultId) { + unwrap(std::move(res)); + throw Error( + "Wasm function '%s' from '%s' finished without returning a value", + functionName, + instance.pre->name); + } + + auto & vRes = instance.getValue(instance.resultId); + state.forceValue(vRes, pos); + v = vRes; + } else { + // FIXME: use the "start" function if present. + instance.runFunction("nix_wasm_init_v1", {}); + + auto res = instance.runFunction(functionName, {(int32_t) argId}); + if (res.size() != 1) + throw Error( + "Wasm function '%s' from '%s' did not return exactly one value", functionName, instance.pre->name); + if (res[0].kind() != ValKind::I32) + throw Error( + "Wasm function '%s' from '%s' did not return an i32 value", functionName, instance.pre->name); + auto & vRes = instance.getValue(res[0].i32()); + state.forceValue(vRes, pos); + v = vRes; + } + } catch (Error & e) { + e.addTrace(state.positions[pos], "while executing a Wasm module"); + throw; + } +} + +static RegisterPrimOp primop_wasm( + {.name = "__wasm", + .args = {"config", "arg"}, + .doc = R"( + Call a Wasm function with the specified argument. + + The first argument must be an attribute set with the following attributes: + - `path`: Path to the Wasm module (mutually exclusive with `wat`) + - `wat`: WebAssembly Text format source as a string (mutually exclusive with `path`) + - `function`: Function name to call (required for non-WASI modules, not allowed for WASI modules) + + Exactly one of `path` or `wat` must be specified. + + The second argument is the value to pass to the function. + + WASI mode is automatically enabled if the module imports from `wasi_snapshot_preview1`. + + Example (non-WASI): + ```nix + builtins.wasm { + path = ./foo.wasm; + function = "fib"; + } 33 + ``` + + Example (reading from a WAT file): + ```nix + builtins.wasm { + wat = builtins.readFile ./fib.wat; + function = "fib"; + } 10 + ``` + + Example (WASI): + ```nix + builtins.wasm { + path = ./bar.wasm; + } { x = 42; } + ``` + )", + .impl = prim_wasm, + .experimentalFeature = Xp::WasmBuiltin}); + +} // namespace nix diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ed91cad85a47..1912262ee576 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -20,9 +20,13 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set"; + str << "«failed»"; break; case nFunction: if (v.isLambda()) { diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index f2f62a636982..57441fa379a7 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -249,7 +249,11 @@ class Printer void printString(Value & v) { - printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors); + NixStringContext context; + copyContext(v, context); + std::ostringstream s; + printLiteralString(s, v.string_view(), options.maxStringLength, options.ansiColors); + output << state.devirtualize(s.str(), context); } void printPath(Value & v) @@ -498,7 +502,7 @@ class Printer output << "«potential infinite recursion»"; if (options.ansiColors) output << ANSI_NORMAL; - } else if (v.isThunk() || v.isApp()) { + } else if (!v.isFinished()) { if (options.ansiColors) output << ANSI_MAGENTA; output << "«thunk»"; @@ -515,7 +519,7 @@ class Printer output << ANSI_MAGENTA; // Historically, a tried and then ignored value (e.g. through tryEval) was // reverted to the original thunk. - output << "«thunk»"; + output << "«failed»"; if (options.ansiColors) output << ANSI_NORMAL; } diff --git a/src/libexpr/provenance.cc b/src/libexpr/provenance.cc new file mode 100644 index 000000000000..8bce4f120763 --- /dev/null +++ b/src/libexpr/provenance.cc @@ -0,0 +1,25 @@ +#include "nix/expr/provenance.hh" +#include "nix/util/json-utils.hh" + +#include + +namespace nix { + +nlohmann::json DerivationProvenance::to_json() const +{ + return nlohmann::json{ + {"type", "derivation"}, + {"meta", *meta}, + {"next", next ? next->to_json() : nlohmann::json(nullptr)}, + }; +} + +Provenance::Register registerDerivationProvenance("derivation", [](nlohmann::json json) { + auto & obj = getObject(json); + std::shared_ptr next; + if (auto p = optionalValueAt(obj, "next"); p && !p->is_null()) + next = Provenance::from_json(*p); + return make_ref(next, make_ref(valueAt(obj, "meta"))); +}); + +} // namespace nix diff --git a/src/libexpr/root-value.cc b/src/libexpr/root-value.cc new file mode 100644 index 000000000000..7ad2c645db60 --- /dev/null +++ b/src/libexpr/root-value.cc @@ -0,0 +1,82 @@ +#include "nix/expr/root-value.hh" +#include "nix/expr/eval-gc.hh" +#include "nix/util/sync.hh" + +namespace nix { + +#if NIX_USE_BOEHMGC + +namespace { +/** + * A root value slot: either in use (rooting a value) or on the + * freelist. Slots are carved out of uncollectable slabs, which are + * permanently part of the GC root set. This avoids doing a + * GC_MALLOC_UNCOLLECTABLE() / GC_FREE() pair per root value, which + * requires taking the global GC allocation lock — a significant + * source of contention during parallel evaluation. + * + * GC safety: the slabs are conservatively scanned. In-use slots + * contain a `Value *`, which roots the value. Free slots contain a + * pointer to the next free slot (or null), i.e. a pointer into a + * slab, which is scanned harmlessly since slabs are never freed + * anyway. + */ +union Slot +{ + Value * value; + Slot * nextFree; +}; + +static_assert(sizeof(Slot) == sizeof(Value *)); +} // namespace + +/* Head of the freelist of slots. Never destroyed since root values + held by statics may be released after us during shutdown. */ +static auto & freeSlots = *new Sync{nullptr}; + +#endif + +RootValue::RootValue(Value * v) +{ +#if NIX_USE_BOEHMGC + Slot * slot; + { + auto head(freeSlots.lock()); + if (!*head) { + constexpr size_t slabSize = 4096; + auto slab = (Slot *) GC_MALLOC_UNCOLLECTABLE(slabSize * sizeof(Slot)); + if (!slab) + throw std::bad_alloc(); + for (size_t i = 0; i + 1 < slabSize; ++i) + slab[i].nextFree = &slab[i + 1]; + slab[slabSize - 1].nextFree = nullptr; + *head = slab; + } + slot = *head; + *head = slot->nextFree; + } + + slot->value = v; + + this->slot = &slot->value; +#else + this->slot = new Value *(v); +#endif +} + +void RootValue::freeRootValueSlot() +{ +#if NIX_USE_BOEHMGC + /* Note: writing `nextFree` overwrites the `Value *`, so this also + stops the slot from keeping the value alive. */ + auto s = reinterpret_cast(slot); + auto head(freeSlots.lock()); + s->nextFree = *head; + *head = s; +#else + delete slot; +#endif + slot = nullptr; +} + +} // namespace nix diff --git a/src/libexpr/symbol-table.cc b/src/libexpr/symbol-table.cc new file mode 100644 index 000000000000..052c72570371 --- /dev/null +++ b/src/libexpr/symbol-table.cc @@ -0,0 +1,63 @@ +#include "nix/expr/symbol-table.hh" +#include "nix/util/logging.hh" + +#include + +namespace nix { + +#ifndef MAP_NORESERVE +# define MAP_NORESERVE 0 +#endif + +static void * allocateLazyMemory(size_t maxSize) +{ + auto p = mmap(nullptr, maxSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) + throw SysError("allocating arena using mmap"); + return p; +} + +ContiguousArena::ContiguousArena(size_t maxSize) + : data((char *) allocateLazyMemory(maxSize)) + , maxSize(maxSize) +{ +} + +size_t ContiguousArena::allocate(size_t bytes) +{ + auto offset = size.fetch_add(bytes); + if (offset + bytes > maxSize) + throw Error("arena ran out of space"); + return offset; +} + +Symbol SymbolTable::create(std::string_view s) +{ + uint32_t idx; + + auto visit = [&](const SymbolStr & sym) { idx = ((const char *) sym.s) - arena.data; }; + + symbols.insert_and_visit(SymbolStr::Key{s, arena}, visit, visit); + + return Symbol(idx); +} + +SymbolStr::SymbolStr(const SymbolStr::Key & key) +{ + auto size = SymbolStr::computeSize(key.s); + + auto id = key.arena.allocate(size); + + auto v = (SymbolValue *) (const_cast(key.arena.data) + id); + + auto s = (StringData *) (v + 1); + s->size_ = key.s.size(); + std::memcpy(s->data_, key.s.data(), key.s.size()); + s->data_[key.s.size()] = '\0'; + + v->mkStringNoCopy(*s); + + this->s = v; +} + +} // namespace nix diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4fac29a6671d..134417e165d6 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -2,109 +2,150 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/store-api.hh" #include "nix/util/signals.hh" +#include "nix/expr/parallel-eval.hh" #include -#include #include namespace nix { + using json = nlohmann::json; +#pragma GCC diagnostic ignored "-Wswitch-enum" + +static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos) +{ + state.forceValue(v, pos); + + Executor::WorkItems work; + + switch (v.type()) { + + case nAttrs: { + NixStringContext context; + if (state.tryAttrsToString(pos, v, context, false, false)) + return; + if (v.attrs()->get(state.s.outPath)) + return; + for (auto & a : *v.attrs()) + state.addWork( + work, 0, [value(RootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }); + break; + } + + default: + break; + } + + state.executor->spawn(std::move(work)); +} + // TODO: rename. It doesn't print. json printValueAsJSON( EvalState & state, bool strict, Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { - checkInterrupt(); + if (strict && state.executor->enabled && !Executor::amWorkerThread) + parallelForceDeep(state, v, pos); - auto _level = state.addCallDepth(pos); + auto recurse = [&](this const auto & recurse, json & res, Value & v, PosIdx pos) -> void { + checkInterrupt(); - if (strict) - state.forceValue(v, pos); + auto _level = state.addCallDepth(pos); - json out; + if (strict) + state.forceValue(v, pos); - switch (v.type()) { + switch (v.type()) { - case nInt: - out = v.integer().value; - break; + case nInt: + res = v.integer().value; + break; - case nBool: - out = v.boolean(); - break; + case nBool: + res = v.boolean(); + break; - case nString: - copyContext(v, context); - out = v.string_view(); - break; + case nString: { + copyContext(v, context); + res = v.string_view(); + break; + } - case nPath: - if (copyToStore) - out = state.store->printStorePath(state.copyPathToStore(context, v.path())); - else - out = v.path().path.abs(); - break; + case nPath: + if (copyToStore) + res = state.store->printStorePath(state.copyPathToStore(context, v.path(), v.determinePos(pos))); + else + res = v.path().path.abs(); + break; - case nNull: - // already initialized as null - break; + case nNull: + // already initialized as null + break; - case nAttrs: { - auto maybeString = state.tryAttrsToString(pos, v, context, false, false); - if (maybeString) { - out = *maybeString; + case nAttrs: { + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); + if (maybeString) { + res = *maybeString; + break; + } + if (auto i = v.attrs()->get(state.s.outPath)) + return recurse(res, *i->value, i->pos); + else { + res = json::object(); + for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { + json & j = res.emplace(state.symbols[a->name], json()).first.value(); + try { + recurse(j, *a->value, a->pos); + } catch (Error & e) { + e.addTrace( + state.positions[a->pos], + HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); + throw; + } + } + } break; } - if (auto i = v.attrs()->get(state.s.outPath)) - return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); - else { - out = json::object(); - for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { + + case nList: { + res = json::array(); + for (const auto & [i, elem] : enumerate(v.listView())) { try { - out.emplace( - state.symbols[a->name], - printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); + res.push_back(json()); + recurse(res.back(), *elem, pos); } catch (Error & e) { - e.addTrace( - state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); + e.addTrace(state.positions[pos], HintFmt("while evaluating list element at index %1%", i)); throw; } } + break; } - break; - } - case nList: { - out = json::array(); - int i = 0; - for (auto elem : v.listView()) { - try { - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); - } catch (Error & e) { - e.addTrace(state.positions[pos], HintFmt("while evaluating list element at index %1%", i)); - throw; - } - i++; + case nExternal: { + res = v.external()->printValueAsJSON(state, strict, context, copyToStore); + break; } - break; - } - case nExternal: - return v.external()->printValueAsJSON(state, strict, context, copyToStore); - break; + case nFloat: + res = v.fpoint(); + break; - case nFloat: - out = v.fpoint(); - break; + case nThunk: + case nFailed: + case nFunction: + state.error("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow(); + } + }; - case nThunk: - case nFailed: - case nFunction: - state.error("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow(); - } - return out; + json res; + + recurse(res, v, pos); + + return res; } +void JSONSerializationError::anchor() {} + void printValueAsJSON( EvalState & state, bool strict, diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 4a17bbdc3a83..13b346af11fc 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -2,10 +2,10 @@ #include "nix/expr/value/context.hh" #include "nix/store/store-dir-config.hh" -#include - namespace nix { +void BadNixStringContextElem::anchor() {} + NixStringContextElem NixStringContextElem::parse(std::string_view s0, const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; @@ -51,6 +51,11 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0, const Expe .drvPath = StorePath{s.substr(1)}, }; } + case '@': { + return NixStringContextElem::Path{ + .storePath = StorePath{s.substr(1)}, + }; + } default: { // Ensure no '!' if (s.find("!") != std::string_view::npos) { @@ -91,6 +96,10 @@ std::string NixStringContextElem::to_string() const res += '='; res += d.drvPath.to_string(); }, + [&](const NixStringContextElem::Path & p) { + res += '@'; + res += p.storePath.to_string(); + }, }, raw); @@ -108,6 +117,9 @@ std::string NixStringContextElem::display(const StoreDirConfig & store) const return store.printStorePath(d.drvPath) + " (deep)"; }, [&](const NixStringContextElem::Built & b) -> std::string { return SingleDerivedPath{b}.to_string(store); }, + [&](const NixStringContextElem::Path & p) -> std::string { + return store.printStorePath(p.storePath) + " (untracked)"; + }, }, raw); } diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index db415d9173e7..44d0e7c31876 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,16 +50,32 @@ headers += files('nix_api_fetchers.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixfetchersc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixfetchersc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libfetchers-c/meson.options b/src/libfetchers-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libfetchers-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libfetchers-c/nix_api_fetchers.cc b/src/libfetchers-c/nix_api_fetchers.cc index 7fefedb0c70f..2b3dd56631c5 100644 --- a/src/libfetchers-c/nix_api_fetchers.cc +++ b/src/libfetchers-c/nix_api_fetchers.cc @@ -7,7 +7,7 @@ extern "C" { nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context) { try { - auto fetchersSettings = nix::make_ref(nix::fetchers::Settings{}); + auto fetchersSettings = nix::make_ref(); return new nix_fetchers_settings{ .settings = fetchersSettings, }; diff --git a/src/libfetchers-c/package.nix b/src/libfetchers-c/package.nix index 9a601d70417c..39147228f2c0 100644 --- a/src/libfetchers-c/package.nix +++ b/src/libfetchers-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -17,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-fetchers-c"; + pname = "determinate-nix-fetchers-c"; inherit version; workDir = ./.; @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 7127434db9df..a482b70dcecf 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -3,8 +3,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/tests/characterization.hh" namespace nix::fetchers { @@ -15,10 +13,7 @@ class AccessKeysTest : public ::testing::Test protected: public: - void SetUp() override - { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - } + void SetUp() override {} void TearDown() override {} }; diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 000000000000..4d0cbbb9ef74 --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git-lfs-fetch.cc b/src/libfetchers-tests/git-lfs-fetch.cc new file mode 100644 index 000000000000..d01a43151df5 --- /dev/null +++ b/src/libfetchers-tests/git-lfs-fetch.cc @@ -0,0 +1,28 @@ +#include "nix/fetchers/git-lfs-fetch.hh" +#include "nix/util/url.hh" + +#include + +namespace nix::lfs { + +struct GitLFSParameterizedTestFixture : public ::testing::TestWithParam> +{}; + +TEST_P(GitLFSParameterizedTestFixture, get_lfs_api) +{ + auto & [input, expected] = GetParam(); + ASSERT_EQ(getLfsApi(parseURL(input)).endpoint, expected); +}; + +INSTANTIATE_TEST_SUITE_P( + GitLFSTests, + GitLFSParameterizedTestFixture, + ::testing::Values( + std::pair{"https://git-server.com/foo/bar", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar.git", "https://git-server.com/foo/bar.git/info/lfs"}, + std::pair{"https://git-server.com", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com/", "https://git-server.com/.git/info/lfs"}, + std::pair{"https://git-server.com//", "https://git-server.com//.git/info/lfs"}, + std::pair{"https://git-server.com/foo/bar/", "https://git-server.com/foo/bar.git/info/lfs"})); + +} // namespace nix::lfs diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 0b21fd0c67d5..580769936d41 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -10,12 +10,11 @@ #include #include "nix/util/fs-sink.hh" #include "nix/util/serialise.hh" -#include "nix/fetchers/git-lfs-fetch.hh" #include #include -namespace nix { +namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { @@ -234,4 +233,4 @@ TEST(GitUtils, isLegalRefName) ASSERT_FALSE(isLegalRefName("")); } -} // namespace nix +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index abc3dd74c5c6..e41bbe520f41 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -1,9 +1,9 @@ -#include "nix/store/store-open.hh" #include "nix/store/globals.hh" #include "nix/store/dummy-store.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/util/url.hh" #include #include @@ -97,7 +97,7 @@ static void commitAll(git_repository * repo, const char * msg) } // namespace -using namespace nix; +namespace nix::fetchers { class GitTest : public ::testing::Test { @@ -190,7 +190,7 @@ TEST_F(GitTest, submodulePeriodSupport) auto input = fetchers::Input::fromAttrs( settings, { - {"url", "file://" + repoPath.string()}, + {"url", "file://" + encodeUrlPath(pathToUrlPath(repoPath))}, {"submodules", Explicit{true}}, {"type", "git"}, {"ref", "main"}, @@ -200,3 +200,5 @@ TEST_F(GitTest, submodulePeriodSupport) ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); } + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/input.cc b/src/libfetchers-tests/input.cc index faff55f2c2d0..170ff4485c7b 100644 --- a/src/libfetchers-tests/input.cc +++ b/src/libfetchers-tests/input.cc @@ -1,8 +1,12 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/tests/gmock-matchers.hh" +#include "nix/util/url.hh" #include +#include #include @@ -58,4 +62,18 @@ INSTANTIATE_TEST_SUITE_P( }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +namespace fetchers { + +class GitHubInputTest : public ::testing::Test +{}; + +TEST_F(GitHubInputTest, throwOnInvalidURLParam) +{ + EXPECT_THAT( + []() { Input::fromURL(fetchers::Settings{}, "github:a/b?tag=foo"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("tag"))); +} + +} // namespace fetchers + } // namespace nix diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 6bccdb05c9a1..22cd6b378302 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -40,6 +40,8 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', + 'git-lfs-fetch.cc', 'git-utils.cc', 'git.cc', 'input.cc', @@ -66,6 +68,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libfetchers-tests/nix_api_fetchers.cc b/src/libfetchers-tests/nix_api_fetchers.cc index 8f3e6e3c5839..ff2c5d8ff319 100644 --- a/src/libfetchers-tests/nix_api_fetchers.cc +++ b/src/libfetchers-tests/nix_api_fetchers.cc @@ -1,4 +1,3 @@ -#include "gmock/gmock.h" #include #include "nix_api_fetchers.h" diff --git a/src/libfetchers-tests/public-key.cc b/src/libfetchers-tests/public-key.cc index 2991223f6b35..dabbd464424c 100644 --- a/src/libfetchers-tests/public-key.cc +++ b/src/libfetchers-tests/public-key.cc @@ -1,6 +1,5 @@ #include #include "nix/fetchers/fetchers.hh" -#include "nix/util/json-utils.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 841808bd16a9..84c74df275df 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -1,10 +1,21 @@ #include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" #include namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -27,11 +38,15 @@ nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + /* The __final attribute is purely internal, so never serialize it. */ + if (attr.first == "__final") + continue; + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -39,12 +54,23 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -62,7 +88,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -80,7 +107,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -97,11 +125,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/builtin-flake-registry.json b/src/libfetchers/builtin-flake-registry.json new file mode 100644 index 000000000000..8071e8ae7719 --- /dev/null +++ b/src/libfetchers/builtin-flake-registry.json @@ -0,0 +1,436 @@ +{ + "flakes": [ + { + "from": { + "id": "agda", + "type": "indirect" + }, + "to": { + "owner": "agda", + "repo": "agda", + "type": "github" + } + }, + { + "from": { + "id": "agenix", + "type": "indirect" + }, + "to": { + "owner": "ryantm", + "repo": "agenix", + "type": "github" + } + }, + { + "from": { + "id": "arion", + "type": "indirect" + }, + "to": { + "owner": "hercules-ci", + "repo": "arion", + "type": "github" + } + }, + { + "from": { + "id": "blender-bin", + "type": "indirect" + }, + "to": { + "dir": "blender", + "owner": "edolstra", + "repo": "nix-warez", + "type": "github" + } + }, + { + "from": { + "id": "bundlers", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "bundlers", + "type": "github" + } + }, + { + "from": { + "id": "cachix", + "type": "indirect" + }, + "to": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + { + "from": { + "id": "composable", + "type": "indirect" + }, + "to": { + "owner": "ComposableFi", + "repo": "composable", + "type": "github" + } + }, + { + "from": { + "id": "disko", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + { + "from": { + "id": "dreampkgs", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "dreampkgs", + "type": "github" + } + }, + { + "from": { + "id": "dwarffs", + "type": "indirect" + }, + "to": { + "owner": "edolstra", + "repo": "dwarffs", + "type": "github" + } + }, + { + "from": { + "id": "emacs-overlay", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "emacs-overlay", + "type": "github" + } + }, + { + "from": { + "id": "fenix", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + { + "from": { + "id": "flake-parts", + "type": "indirect" + }, + "to": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + { + "from": { + "id": "flake-utils", + "type": "indirect" + }, + "to": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + { + "from": { + "id": "helix", + "type": "indirect" + }, + "to": { + "owner": "helix-editor", + "repo": "helix", + "type": "github" + } + }, + { + "from": { + "id": "hercules-ci-agent", + "type": "indirect" + }, + "to": { + "owner": "hercules-ci", + "repo": "hercules-ci-agent", + "type": "github" + } + }, + { + "from": { + "id": "hercules-ci-effects", + "type": "indirect" + }, + "to": { + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "type": "github" + } + }, + { + "from": { + "id": "home-manager", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + { + "from": { + "id": "hydra", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "hydra", + "type": "github" + } + }, + { + "from": { + "id": "mach-nix", + "type": "indirect" + }, + "to": { + "owner": "DavHau", + "repo": "mach-nix", + "type": "github" + } + }, + { + "from": { + "id": "ngipkgs", + "type": "indirect" + }, + "to": { + "owner": "ngi-nix", + "repo": "ngipkgs", + "type": "github" + } + }, + { + "from": { + "id": "nickel", + "type": "indirect" + }, + "to": { + "owner": "tweag", + "repo": "nickel", + "type": "github" + } + }, + { + "from": { + "id": "nix", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, + { + "from": { + "id": "nix-darwin", + "type": "indirect" + }, + "to": { + "owner": "nix-darwin", + "repo": "nix-darwin", + "type": "github" + } + }, + { + "from": { + "id": "nix-serve", + "type": "indirect" + }, + "to": { + "owner": "edolstra", + "repo": "nix-serve", + "type": "github" + } + }, + { + "from": { + "id": "nixops", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "nixops", + "type": "github" + } + }, + { + "from": { + "id": "nixos-anywhere", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "nixos-anywhere", + "type": "github" + } + }, + { + "from": { + "id": "nixos-hardware", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "nixos-hardware", + "type": "github" + } + }, + { + "from": { + "id": "nixos-homepage", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "nixos-homepage", + "type": "github" + } + }, + { + "from": { + "id": "nixos-search", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "nixos-search", + "type": "github" + } + }, + { + "exact": true, + "from": { + "id": "nixpkgs", + "type": "indirect" + }, + "to": { + "type": "tarball", + "url": "https://flakehub.com/f/DeterminateSystems/nixpkgs-weekly/0.1" + } + }, + { + "from": { + "id": "nixpkgs", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + { + "from": { + "id": "nur", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "NUR", + "type": "github" + } + }, + { + "from": { + "id": "patchelf", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "patchelf", + "type": "github" + } + }, + { + "from": { + "id": "poetry2nix", + "type": "indirect" + }, + "to": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + { + "from": { + "id": "pridefetch", + "type": "indirect" + }, + "to": { + "owner": "SpyHoodle", + "repo": "pridefetch", + "type": "github" + } + }, + { + "from": { + "id": "sops-nix", + "type": "indirect" + }, + "to": { + "owner": "Mic92", + "repo": "sops-nix", + "type": "github" + } + }, + { + "from": { + "id": "systems", + "type": "indirect" + }, + "to": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + { + "from": { + "id": "templates", + "type": "indirect" + }, + "to": { + "owner": "NixOS", + "repo": "templates", + "type": "github" + } + } + ], + "version": 2 +} diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc new file mode 100644 index 000000000000..44b3baf0b1aa --- /dev/null +++ b/src/libfetchers/builtin.cc @@ -0,0 +1,60 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/archive.hh" +#include "nix/store/filetransfer.hh" +#include "nix/store/store-open.hh" + +#include + +namespace nix { + +static void builtinFetchTree(const BuiltinBuilderContext & ctx) +{ + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); + + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-tree' requires an 'out' output"); + + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) + throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation"); + + if (!ctx.drv.structuredAttrs) + throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + + setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1); + + using namespace fetchers; + + fetchers::Settings myFetchSettings; + myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + + // Make sure we don't use the FileTransfer object of the parent + // since it's in a broken state after the fork. We also must not + // delete it, so hang on to the shared_ptr. + // FIXME: move FileTransfer into fetchers::Settings. + static auto prevFileTransfer = resetFileTransfer(); + + // FIXME: disable use of the git/tarball cache + + auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs(ctx.drv.structuredAttrs->structuredAttrs.at("input"))); + + std::cerr << fmt("fetching '%s'...\n", input.to_string()); + + /* Functions like downloadFile() expect a store. We can't use the + real one since we're in a forked process. FIXME: use recursive + Nix's daemon so we can use the real store? */ + auto tmpStore = openStore(ctx.tmpDirInSandbox + "/nix"); + + auto [accessor, lockedInput] = input.getAccessor(myFetchSettings, *tmpStore); + + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); + + restorePath(ctx.outputs.at("out"), *source); +} + +static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree); + +} // namespace nix diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index cf60e29a8e15..0c24b1072a4d 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -26,6 +26,8 @@ create table if not exists Cache ( struct CacheImpl : Cache { + void anchor() override; + struct State { SQLite db; @@ -60,7 +62,11 @@ struct CacheImpl : Cache void upsert(const Key & key, const Attrs & value) override { _state.lock() - ->upsert.use()(key.first)(attrsToJSON(key.second).dump())(attrsToJSON(value).dump())(time(nullptr)) + ->upsert.use() + .apply(key.first) + .apply(attrsToJSON(key.second).dump()) + .apply(attrsToJSON(value).dump()) + .apply(time(nullptr)) .exec(); } @@ -87,7 +93,7 @@ struct CacheImpl : Cache auto keyJSON = attrsToJSON(key.second).dump(); - auto stmt(state->lookup.use()(key.first)(keyJSON)); + auto stmt(state->lookup.use().apply(key.first).apply(keyJSON)); if (!stmt.next()) { debug("did not find cache entry for '%s:%s'", key.first, keyJSON); return {}; @@ -115,7 +121,7 @@ struct CacheImpl : Cache upsert(key, value); } - std::optional lookupStorePath(Key key, Store & store) override + std::optional lookupStorePath(Key key, Store & store, bool allowInvalid) override { key.second.insert_or_assign("store", store.storeDir); @@ -129,7 +135,7 @@ struct CacheImpl : Cache ResultWithStorePath res2(*res, StorePath(storePathS)); store.addTempRoot(res2.storePath); - if (!store.isValidPath(res2.storePath)) { + if (!allowInvalid && !store.isValidPath(res2.storePath)) { // FIXME: we could try to substitute 'storePath'. debug( "ignoring disappeared cache entry '%s:%s' -> '%s'", @@ -151,11 +157,15 @@ struct CacheImpl : Cache std::optional lookupStorePathWithTTL(Key key, Store & store) override { - auto res = lookupStorePath(std::move(key), store); + auto res = lookupStorePath(std::move(key), store, false); return res && !res->expired ? res : std::nullopt; } }; +void Cache::anchor() {} + +void CacheImpl::anchor() {} + ref Settings::getCache() const { auto cache(_cache.lock()); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index f92b94a0b3bd..568e805a0b57 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,7 +1,18 @@ #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/config-global.hh" namespace nix::fetchers { Settings::Settings() {} +void Settings::anchor() {} + } // namespace nix::fetchers + +namespace nix { + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} // namespace nix diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 3af2d4c83e88..4062d5899a4b 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -3,8 +3,23 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/environment-variables.hh" +#include + namespace nix { +struct SrcToStore +{ + boost::concurrent_flat_map< + std::tuple, + std::tuple> + cache; +}; + +ref fetchers::Settings::createSrcToStore() +{ + return make_ref(); +} + fetchers::Cache::Key makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path) { @@ -36,6 +51,14 @@ std::pair fetchToStore2( PathFilter * filter, RepairFlag repair) { + auto srcToStoreKey = std::make_tuple(path, method.raw, std::string(name)); + + if (!filter) { + auto dstPathCached = getConcurrent(settings.srcToStore->cache, srcToStoreKey); + if (dstPathCached && (mode == FetchMode::DryRun || std::get<2>(*dstPathCached) == FetchMode::Copy)) + return std::make_pair(std::get<0>(*dstPathCached), std::get<1>(*dstPathCached)); + } + std::optional cacheKey; auto [subpath, fingerprint] = filter ? std::pair>{path.path, std::nullopt} @@ -53,21 +76,21 @@ std::pair fetchToStore2( if (mode != FetchMode::DryRun) store.addTempRoot(storePath); - if (mode == FetchMode::DryRun || store.isValidPath(storePath)) { + if (mode == FetchMode::DryRun || store.maybeQueryPathInfo(storePath)) { debug( "source path '%s' cache hit in '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); return {storePath, hash}; } debug("source path '%s' not in store", path); } } else { static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; - if (barf && !filter) + if (barf && !filter && !(path.to_string().starts_with("/") || path.to_string().starts_with("«path:/"))) throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); - // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); } @@ -82,6 +105,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( @@ -102,8 +126,9 @@ std::pair fetchToStore2( throw Error("path '%s' lacks a CA field", store.printStorePath(storePath)); info->ca->hash; }); - debug( - "copied '%s' to '%s' (hash '%s')", + printMsg( + lvlChatty, + "copied source '%s' -> '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); @@ -113,6 +138,9 @@ std::pair fetchToStore2( if (cacheKey) settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); + if (!filter) + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); + return {storePath, hash}; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 575e479d49e3..b1bc134ac34e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,14 +1,19 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" +#include "nix/util/fs-sink.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" #include "nix/fetchers/fetch-settings.hh" -#include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/provenance.hh" #include "nix/util/url.hh" -#include "nix/util/archive.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" +#include #include +#include namespace nix::fetchers { @@ -126,24 +131,30 @@ std::optional Input::getFingerprint(Store & store) const return fingerprint; } -ParsedURL Input::toURL() const +ParsedURL Input::toURL(bool abbreviate) const { if (!scheme) throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs)); - return scheme->toURL(*this); + + auto url = scheme->toURL(*this, abbreviate); + + if (abbreviate) + url.query.erase("narHash"); + + return url; } -std::string Input::toURLString(const StringMap & extraQuery) const +std::string Input::toURLString(const StringMap & extraQuery, bool abbreviate) const { - auto url = toURL(); + auto url = toURL(abbreviate); for (auto & attr : extraQuery) url.query.insert(attr); return url.to_string(); } -std::string Input::to_string() const +std::string Input::to_string(bool abbreviate) const { - return toURL().to_string(); + return toURL(abbreviate).to_string(); } bool Input::isDirect() const @@ -163,8 +174,7 @@ bool Input::isFinal() const std::optional Input::isRelative() const { - assert(scheme); - return scheme->isRelative(*this); + return scheme ? scheme->isRelative(*this) : std::nullopt; } Attrs Input::toAttrs() const @@ -189,36 +199,30 @@ bool Input::contains(const Input & other) const return false; } -// FIXME: remove -std::pair Input::fetchToStore(const Settings & settings, Store & store) const +std::tuple, Input> Input::fetchToStore(const Settings & settings, Store & store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - auto [storePath, input] = [&]() -> std::pair { - try { - auto [accessor, result] = getAccessorUnchecked(settings, store); - - auto storePath = - nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); + try { + auto [accessor, result] = getAccessorUnchecked(settings, store); - auto narHash = store.queryPathInfo(storePath)->narHash; - result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + auto storePath = nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); - result.attrs.insert_or_assign("__final", Explicit(true)); + auto narHash = store.queryPathInfo(storePath)->narHash; + result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - assert(result.isFinal()); + result.attrs.insert_or_assign("__final", Explicit(true)); - checkLocks(*this, result); + assert(result.isFinal()); - return {storePath, result}; - } catch (Error & e) { - e.addTrace({}, "while fetching the input '%s'", to_string()); - throw; - } - }(); + checkLocks(*this, result); - return {std::move(storePath), input}; + return {std::move(storePath), accessor, result}; + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } } void Input::checkLocks(Input specified, Input & result) @@ -272,24 +276,10 @@ void Input::checkLocks(Input specified, Input & result) } } - if (auto prevLastModified = specified.getLastModified()) { - if (result.getLastModified() != prevLastModified) - throw Error( - "'lastModified' attribute mismatch in input '%s', expected %d, got %d", - result.to_string(), - *prevLastModified, - result.getLastModified().value_or(-1)); - } - if (auto prevRev = specified.getRev()) { if (result.getRev() != prevRev) throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); } - - if (auto prevRevCount = specified.getRevCount()) { - if (result.getRevCount() != prevRevCount) - throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount); - } } std::pair, Input> Input::getAccessor(const Settings & settings, Store & store) const @@ -315,55 +305,96 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* The tree may already be in the Nix store, or it could be - substituted (which is often faster than fetching from the - original source). So check that. We only do this for final - inputs, otherwise there is a risk that we don't return the - same attributes (like `lastModified`) that the "real" fetcher - would return. - - FIXME: add a setting to disable this. - FIXME: substituting may be slower than fetching normally, - e.g. for fetchers like Git that are incremental! - */ - if (isFinal() && getNarHash()) { - try { - auto storePath = computeStorePath(store); + std::optional storePath; + if (isFinal() && getNarHash()) + storePath = computeStorePath(store); + + auto makeStoreAccessor = [&]() -> std::pair, Input> { + auto accessor = store.requireStoreObjectAccessor(*storePath); + + // FIXME: use the NAR hash for fingerprinting Git trees since it may have a .gitattributes file and we don't + // know if we used `git archive` or libgit2 to fetch it. + accessor->fingerprint = getType() == "git" ? std::optional(storePath->hashPart()) : getFingerprint(store); + cachedFingerprint = accessor->fingerprint; + + // Store a cache entry for the substituted tree so later fetches + // can reuse the existing nar instead of copying the unpacked + // input back into the store on every evaluation. + if (accessor->fingerprint) { + settings.getCache()->upsert( + makeSourcePathToHashCacheKey( + *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), + {{"hash", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)}}); + } + + accessor->provenance = std::make_shared(*this); + + // FIXME: ideally we would use the `showPath()` of the + // "real" accessor for this fetcher type. + accessor->setPathDisplay("«" + to_string(true) + "»"); + + return {accessor, *this}; + }; + + /* If a tree with the expected hash is already in the Nix store, + reuse it. We only do this for final inputs, since otherwise + there is a risk that we don't return the same attributes (like + `lastModified`) that the "real" fetcher would return. */ + if (storePath) { + store.addTempRoot(*storePath); + if (store.isValidPath(*storePath)) { + debug("using input '%s' in '%s'", to_string(), store.printStorePath(*storePath)); + return makeStoreAccessor(); + } + } - store.ensurePath(storePath); + auto fixupAccessor = [&](ref accessor, Input result) -> std::pair, Input> { + if (auto fp = accessor->getFingerprint(CanonPath::root).second) + result.cachedFingerprint = *fp; + else + accessor->fingerprint = result.getFingerprint(store); - debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); + accessor->provenance = std::make_shared(result); - auto accessor = store.requireStoreObjectAccessor(storePath); + return {accessor, result}; + }; - accessor->fingerprint = getFingerprint(store); + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + std::filesystem::create_directories(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); - // Store a cache entry for the substituted tree so later fetches - // can reuse the existing nar instead of copying the unpacked - // input back into the store on every evaluation. - if (accessor->fingerprint) { - settings.getCache()->upsert( - makeSourcePathToHashCacheKey( - *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), - {{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}}); - } + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); - accessor->setPathDisplay("«" + to_string() + "»"); + /* See if the input is in the cache of the fetcher. */ + try { + if (auto res = scheme->getAccessor(settings, store, *this, true)) + return fixupAccessor(res->first, std::move(res->second)); + } catch (...) { + } - return {accessor, *this}; - } catch (Error & e) { - debug("substitution of input '%s' failed: %s", to_string(), e.what()); + /* If not, try to substitute the input. */ + if (storePath) { + try { + store.ensurePath(*storePath); + return makeStoreAccessor(); + } + // Ignore any substitution error. + catch (Error & e2) { + debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg); + } catch (...) { } } + /* If we can't substitute, then fetch normally. */ auto [accessor, result] = scheme->getAccessor(settings, store, *this); - - if (!accessor->fingerprint) - accessor->fingerprint = result.getFingerprint(store); - else - result.cachedFingerprint = accessor->fingerprint; - - return {accessor, std::move(result)}; + return fixupAccessor(accessor, result); } Input Input::applyOverrides(std::optional ref, std::optional rev) const @@ -375,19 +406,20 @@ Input Input::applyOverrides(std::optional ref, std::optional void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const { - assert(scheme); + if (!scheme) + throw Error("cannot clone unsupported input '%s'", attrsToJSON(attrs)); scheme->clone(settings, store, *this, destDir); } std::optional Input::getSourcePath() const { - assert(scheme); - return scheme->getSourcePath(*this); + return scheme ? scheme->getSourcePath(*this) : std::nullopt; } void Input::putFile(const CanonPath & path, std::string_view contents, std::optional commitMsg) const { - assert(scheme); + if (!scheme) + throw Error("unsupported input '%s' does not support modifying file '%s'", attrsToJSON(attrs), path); return scheme->putFile(*this, path, contents, commitMsg); } @@ -464,7 +496,7 @@ std::optional Input::getLastModified() const return {}; } -ParsedURL InputScheme::toURL(const Input & input) const +ParsedURL InputScheme::toURL(const Input & input, bool abbreviate) const { throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 0daedcf7c78f..c881718fe5a4 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" +#include "nix/util/sync.hh" -#include +#include namespace nix { @@ -61,9 +62,11 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) +std::shared_ptr FilteringSourceAccessor::getProvenance(const CanonPath & path) { - next->invalidateCache(prefix / path); + if (provenance) + return SourceAccessor::getProvenance(path); + return next->getProvenance(prefix / path); } void FilteringSourceAccessor::checkAccess(const CanonPath & path) @@ -74,39 +77,42 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { - std::set allowedPrefixes; - boost::unordered_flat_set allowedPaths; +private: + void anchor() override {}; +public: + SharedSync> allowedPrefixes; + boost::concurrent_flat_set allowedPaths; AllowListSourceAccessorImpl( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) - , allowedPrefixes(std::move(allowedPrefixes)) - , allowedPaths(std::move(allowedPaths)) + , allowedPrefixes(allowedPrefixes.begin(), allowedPrefixes.end()) + , allowedPaths(allowedPaths.begin(), allowedPaths.end()) { } bool isAllowed(const CanonPath & path) override { - return allowedPaths.contains(path) || path.isAllowed(allowedPrefixes); + /* Read lock is held for the duration of the full expression if the || doesn't short-circuit. */ + return allowedPaths.contains(path) || path.isAllowed(*allowedPrefixes.readLock()); } void allowPrefix(CanonPath prefix) override { - allowedPrefixes.insert(std::move(prefix)); + allowedPrefixes.lock()->insert(std::move(prefix)); } }; ref AllowListSourceAccessor::create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) { - return make_ref( - next, std::move(allowedPrefixes), std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(next, allowedPrefixes, allowedPaths, std::move(makeNotAllowedError)); } bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 4585e68e58ee..c80732a0c21f 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -1,13 +1,17 @@ #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/filetransfer.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" #include "nix/util/util.hh" #include "nix/util/hash.hh" +#include "nix/util/json-utils.hh" #include "nix/store/ssh.hh" +#include "nix/util/deleter.hh" #include #include @@ -21,9 +25,7 @@ namespace nix::lfs { static void downloadToSink( const std::string & url, const std::optional & authHeader, - // FIXME: passing a StringSink is superfluous, we may as well - // return a string. Or use an abstract Sink for streaming. - StringSink & sink, + Sink & sink, std::string sha256Expected, size_t sizeExpected) { @@ -32,29 +34,25 @@ static void downloadToSink( if (authHeader.has_value()) headers.push_back({"Authorization", *authHeader}); request.headers = headers; - getFileTransfer()->download(std::move(request), sink); - auto sizeActual = sink.s.length(); - if (sizeExpected != sizeActual) - throw Error("size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, sizeActual); + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(hashSink, sink); - auto sha256Actual = hashString(HashAlgorithm::SHA256, sink.s).to_string(HashFormat::Base16, false); + getFileTransfer()->download(std::move(request), teeSink); + + auto hashResult = hashSink.finish(); + + if (sizeExpected != hashResult.numBytesDigested) + throw Error( + "size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, hashResult.numBytesDigested); + + auto sha256Actual = hashResult.hash.to_string(HashFormat::Base16, false); if (sha256Actual != sha256Expected) throw Error( "hash mismatch while fetching %s: expected sha256:%s but got sha256:%s", url, sha256Expected, sha256Actual); } -namespace { - -struct LfsApiInfo -{ - std::string endpoint; - std::optional authHeader; -}; - -} // namespace - -static LfsApiInfo getLfsApi(const ParsedURL & url) +LfsApiInfo getLfsApi(ParsedURL url) { assert(url.authority.has_value()); if (url.scheme == "ssh") { @@ -94,7 +92,41 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) return {queryResp.at("href").get(), authIt->get()}; } - return {url.to_string() + "/info/lfs", std::nullopt}; + /** + * Try to mimic what git-lfs will do to plain remotes + * https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md + * + * Try to be smarter with remotes ending in a /, like + * `https://github.com/NixOS/nix/`. This should be + * `https://github.com/NixOS/nix.git/info/lfs`, not + * `https://github.com/NixOS/nix/.git/info/lfs` + */ + bool hasDotGit = false; + for (auto it = url.path.rbegin(); it != url.path.rend(); ++it) { + if (it->empty()) + continue; + if (!it->ends_with(".git")) + *it += ".git"; + hasDotGit = true; + break; + } + if (!hasDotGit) { + if (url.path.size() > 1) // e.g. {"", ""} (single trailing slash) + url.path.back() = ".git"; + else if (url.path.size() == 1) // {""} + url.path.push_back(".git"); + else { // {} + url.path.push_back(""); + url.path.push_back(".git"); + } + } + if (url.path.back().empty()) + url.path.back() = "info"; + else + url.path.push_back("info"); + url.path.push_back("lfs"); + + return {url.to_string(), std::nullopt}; } typedef std::unique_ptr> GitConfig; @@ -253,7 +285,7 @@ std::vector Fetch::fetchUrls(const std::vector & pointe void Fetch::fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const { debug("trying to fetch '%s' using git-lfs", pointerFilePath); @@ -277,9 +309,13 @@ void Fetch::fetch( std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false) + "/" + pointer->oid; auto cachePath = cacheDir / key; - if (pathExists(cachePath)) { + AutoCloseFD cacheFile(openFileReadonly(cachePath, FinalSymlink::DontFollow)); + if (cacheFile) { debug("using cache entry %s -> %s", key, PathFmt(cachePath)); - sink(readFile(cachePath)); + FdSource cacheSource(cacheFile.get()); + auto size = getFileSize(cacheFile.get()); + sizeCallback(size); + cacheSource.drainInto(sink, size); return; } debug("did not find cache entry for %s", key); @@ -290,7 +326,8 @@ void Fetch::fetch( const auto obj = objUrls[0]; try { - std::string sha256 = obj.at("oid"); // oid is also the sha256 + // Use the committed pointer's oid/size for integrity, not server's claim + std::string sha256 = pointer->oid; std::string ourl = obj.at("actions").at("download").at("href"); auto authHeader = [&]() -> std::optional { const auto & download = obj.at("actions").at("download"); @@ -302,14 +339,37 @@ void Fetch::fetch( return std::nullopt; return std::string(*authIt); }(); - const uint64_t size = obj.at("size"); - sizeCallback(size); - downloadToSink(ourl, authHeader, sink, sha256, size); + const uint64_t size = pointer->size; + + auto objOid = getString(valueAt(getObject(obj), "oid")); + auto objSize = getUnsigned(valueAt(getObject(obj), "size")); + if (objOid != pointer->oid || objSize != pointer->size) { + throw Error( + "LFS server returned mismatched oid/size for '%s' (got oid=%s size=%d, expected oid=%s size=%d)", + pointerFilePath, + objOid, + objSize, + pointer->oid, + pointer->size); + } debug("creating cache entry %s -> %s", key, PathFmt(cachePath)); + if (!pathExists(cachePath.parent_path())) createDirs(cachePath.parent_path()); - writeFile(cachePath, sink.s); + auto [tempFile, tempPath] = createTempFile(cachePath.parent_path(), {}); + AutoDelete tempDeleter(tempPath); + FdSink tempSink(tempFile.get()); + downloadToSink(ourl, authHeader, tempSink, sha256, size); + tempSink.flush(); + + std::filesystem::rename(tempPath, cachePath); + tempDeleter.cancel(); + + FdSource cacheSource(tempFile.get()); + cacheSource.restart(); + sizeCallback(size); + cacheSource.drainInto(sink, size); debug("%s fetched with git-lfs", pointerFilePath); } catch (const nlohmann::json::out_of_range & e) { diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 456d869983c2..27476a3e6098 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -13,6 +13,8 @@ #include "nix/util/util.hh" #include "nix/util/thread-pool.hh" #include "nix/util/pool.hh" +#include "nix/util/executable-path.hh" +#include "nix/util/deleter.hh" #include #include @@ -42,9 +44,7 @@ #include #include #include -#include #include -#include #include namespace std { @@ -75,6 +75,8 @@ namespace nix { struct GitSourceAccessor; +namespace { + struct GitError final : public CloneableError { template @@ -98,6 +100,8 @@ struct GitError final : public CloneableError } }; +} // namespace + typedef std::unique_ptr> Repository; typedef std::unique_ptr> TreeEntry; typedef std::unique_ptr> Tree; @@ -227,7 +231,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -427,7 +431,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this ThreadPool pool; - auto process = [&done, &pool, &repoPool](this const auto & process, const git_oid & oid) -> void { + auto process = [&done, &pool, &repoPool](this auto const & process, const git_oid & oid) -> void { auto repo(repoPool.get()); auto _commit = lookupObject(*repo, oid, GIT_OBJECT_COMMIT); @@ -571,7 +575,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; @@ -637,28 +641,46 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // that) // then use code that was removed in this commit (see blame) - auto dir = this->path; - OsStrings gitArgs = { - OS_STR("-C"), - dir.native(), - OS_STR("--git-dir"), - OS_STR("."), - OS_STR("fetch"), - OS_STR("--progress"), - OS_STR("--force"), - }; - if (shallow) { - gitArgs.push_back(OS_STR("--depth")); - gitArgs.push_back(OS_STR("1")); - } - gitArgs.push_back(OS_STR("--")); - gitArgs.push_back(string_to_os_string(url)); - gitArgs.push_back(string_to_os_string(refspec)); + if (ExecutablePath::load().findName("git")) { + auto dir = this->path; + + // Remove shallow.lock left behind by a previously interrupted `git fetch`, as it would prevent `git fetch` + // from running. Note that we already have a repository-wide `PathLock` (see git.cc), so this is safe. + tryUnlink(dir / "shallow.lock"); + + OsStrings gitArgs{"-C", dir.native(), "--git-dir", ".", "fetch", "--progress", "--force"}; + if (shallow) { + gitArgs.push_back(OS_STR("--depth")); + gitArgs.push_back(OS_STR("1")); + } + gitArgs.push_back(OS_STR("--")); + gitArgs.push_back(string_to_os_string(url)); + gitArgs.push_back(string_to_os_string(refspec)); + + auto status = runProgram(RunOptions{.program = "git", .args = gitArgs, .isInteractive = true}).first; + + if (status > 0) + throw Error("Failed to fetch git repository '%s'", url); + } else { + // Fall back to using libgit2 for fetching. This does not + // support SSH very well. + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2{.strings = refspecs, .count = 1}; - auto status = runProgram({.program = "git", .args = gitArgs, .isInteractive = true}).first; + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; - if (status > 0) - throw Error("Failed to fetch git repository '%s'", url); + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + } } void verifyCommit(const Hash & rev, const std::vector & publicKeys) override @@ -774,12 +796,19 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti return make_ref(path, options); } +std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const +{ + return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : ""); +} + /** * Raw git tree input accessor. */ - struct GitSourceAccessor : SourceAccessor { +private: + void anchor() override {}; +public: struct State { ref repo; @@ -798,6 +827,7 @@ struct GitSourceAccessor : SourceAccessor .options = options, }} { + fingerprint = options.makeFingerprint(rev); } void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function sizeCallback) @@ -808,20 +838,16 @@ struct GitSourceAccessor : SourceAccessor if (state->lfsFetch) { if (state->lfsFetch->shouldFetch(path)) { - StringSink s; try { // FIXME: do we need to hold the state lock while // doing this? auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); - state->lfsFetch->fetch(contents, path, s, [&s](uint64_t size) { s.s.reserve(size); }); + state->lfsFetch->fetch(contents, path, sink, sizeCallback); } catch (Error & e) { e.addTrace({}, "while smudging git-lfs file '%s'", path); throw; } - sizeCallback(s.s.size()); - StringSource source{s.s}; - source.drainInto(sink); return; } } @@ -991,7 +1017,7 @@ struct GitSourceAccessor : SourceAccessor { auto entry = lookup(state, path); if (!entry) - throw Error("'%s' does not exist", showPath(path)); + throw FileNotFound("path '%s' does not exist", showPath(path)); return entry; } @@ -1058,6 +1084,9 @@ struct GitSourceAccessor : SourceAccessor struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor { +private: + void anchor() override {}; +public: ref repo; std::optional rev; @@ -1116,6 +1145,10 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor } }; +void GitFileSystemObjectSink::anchor() {} + +namespace { + struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { ref repo; @@ -1411,6 +1444,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink } }; +} // namespace + ref GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options) { auto self = ref(shared_from_this()); @@ -1433,13 +1468,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - std::set{wd.files}, - // Always allow access to the root, but not its children. - boost::unordered_flat_set{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; @@ -1463,6 +1500,7 @@ std::vector> GitRepoImpl::getSubmodules auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + AutoDelete delTemp(pathTemp, /*recursive=*/false); try { writeFull(fdTemp.get(), configS); } catch (SystemError & e) { @@ -1481,6 +1519,7 @@ std::vector> GitRepoImpl::getSubmodules result.push_back({std::move(submodule), *rev}); } + delTemp.deletePath(); return result; } @@ -1493,14 +1532,19 @@ ref Settings::getTarballCache() const * for optimal packfiles. */ static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2"; - return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true}); + auto tarballCache(_tarballCache.lock()); + if (!*tarballCache) + *tarballCache = GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true}); + return ref(*tarballCache); } } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1512,6 +1556,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f6ae63c815ee..f4d9003cb503 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,25 +1,23 @@ +#include "nix/util/environment-variables.hh" #include "nix/util/error.hh" #include "nix/fetchers/fetchers.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" -#include "nix/util/url-parts.hh" #include "nix/store/pathlocks.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/git.hh" #include "nix/fetchers/git-utils.hh" #include "nix/util/logging.hh" -#include "nix/util/finally.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/fetch-to-store.hh" -#include -#include #include #ifndef _WIN32 @@ -32,12 +30,6 @@ namespace nix::fetchers { namespace { -// Explicit initial branch of our bare repo to suppress warnings from new version of git. -// The value itself does not matter, since we always fetch a specific revision or branch. -// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with -// old version of git, which will ignore unrecognized `-c` options. -const std::string gitInitialBranch = "__nix_dummy_branch"; - static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st) { return st.st_mtime + static_cast(settings.tarballTtl) > now; @@ -131,7 +123,7 @@ static std::optional readHeadCached(const Settings & settings, cons std::optional cachedRef; if (st) { cachedRef = readHead(cacheDir); - if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(settings, now, *st)) { + if (cachedRef != std::nullopt && isCacheFileWithinTtl(settings, now, *st)) { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } @@ -174,6 +166,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -407,15 +406,17 @@ struct GitInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); - if (auto ref = input.getRef()) - url.query.insert_or_assign("ref", *ref); + if (auto ref = input.getRef()) { + if (!abbreviate || (*ref != "master" && *ref != "main")) + url.query.insert_or_assign("ref", *ref); + } if (getShallowAttr(input)) url.query.insert_or_assign("shallow", "1"); if (getLfsAttr(input)) @@ -466,7 +467,7 @@ struct GitInputScheme : InputScheme args.push_back(destDir.native()); - runProgram("git", true, args, {}, true); + runProgram("git", true, args, true); } std::optional getSourcePath(const Input & input) const override @@ -526,6 +527,10 @@ struct GitInputScheme : InputScheme }); if (commitMsg) { + auto [tempFd, tempPath] = createTempFile("nix-msg"); + AutoDelete delTemp(tempPath, /*recursive=*/false); + writeFull(tempFd.get(), *commitMsg); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` auto suspension = logger->suspend(); runProgram( @@ -539,9 +544,10 @@ struct GitInputScheme : InputScheme OS_STR("commit"), string_to_os_string(std::string(path.rel())), OS_STR("-F"), - OS_STR("-"), - }, - *commitMsg); + tempPath.native(), + }); + + delTemp.deletePath(); } } } @@ -577,10 +583,10 @@ struct GitInputScheme : InputScheme { if (workdirInfo.isDirty) { if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", locationToArg()); + throw Error("Git tree '%s' has uncommitted changes", locationToArg()); if (settings.warnDirty) - warn("Git tree '%s' is dirty", locationToArg()); + warn("Git tree '%s' has uncommitted changes", locationToArg()); } } @@ -729,14 +735,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -751,6 +755,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -774,7 +790,7 @@ struct GitInputScheme : InputScheme "\n" "To make it visible to Nix, run:\n" "\n" - "git -C %2% add \"%1%\"", + "git -C %2% add -N \"%1%\"", path.rel(), PathFmt(repoPath)); else @@ -797,15 +813,123 @@ struct GitInputScheme : InputScheme } } - std::pair, Input> - getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const + /** + * Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified + * `shallow = true`, or if we already have a `revCount`. + */ + bool canDoShallow(const Input & input) const + { + bool shallow = getShallowAttr(input); + return shallow || input.getRevCount().has_value(); + } + + GitAccessorOptions getGitAccessorOptions(const Input & input) const + { + return GitAccessorOptions{ + .exportIgnore = getExportIgnoreAttr(input), + .smudgeLfs = getLfsAttr(input), + .submodules = getSubmodulesAttr(input), + }; + } + + /** + * Get a `SourceAccessor` for the given Git revision using Nix < 2.20 semantics, i.e. using `git archive` or `git + * checkout`. + */ + ref getLegacyGitAccessor( + const Settings & settings, + Store & store, + RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev, + GitAccessorOptions & options) const + { + if (!options.submodules) + options.exportIgnore = true; + + auto fingerprint = options.makeFingerprint(rev) + ";legacy"; + + auto cacheKey = + makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root); + + auto makeAccessor = [&](const auto & storePath) -> ref { + auto accessor = store.getFSAccessor(storePath); + accessor->fingerprint = fingerprint; + return ref{accessor}; + }; + + if (auto res = settings.getCache()->lookup(cacheKey)) { + auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); + auto storePath = store.makeFixedOutputPathFromCA( + "source", ContentAddressWithReferences::fromParts(ContentAddressMethod::Raw::NixArchive, hash, {})); + store.addTempRoot(storePath); + if (store.maybeQueryPathInfo(storePath)) { + debug("using cached legacy export of revision '%s'", rev.gitRev()); + return makeAccessor(storePath); + } + } + + debug("doing legacy export of revision '%s'", rev.gitRev()); + + auto tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + auto storePath = + options.submodules + ? [&]() { + // Nix < 2.20 used `git checkout` for repos with submodules. + StringSink sink; // don't pollute stdout + runProgram2({.program = "git", .args = {"init", tmpDir, "-b", "master"}, .standardOut = &sink}); + runProgram2( + {.program = "git", + .args = {"-C", tmpDir, "remote", "add", "origin", repoDir}, + .standardOut = &sink}); + runProgram2( + {.program = "git", + .args = {"-C", tmpDir, "fetch", "--quiet", "origin", rev.gitRev()}, + .standardOut = &sink}); + runProgram2( + {.program = "git", + .args = {"-C", tmpDir, "checkout", "--quiet", rev.gitRev()}, + .standardOut = &sink}); + PathFilter filter = [&](const std::string & path) { return baseNameOf(path) != ".git"; }; + return store.addToStore( + "source", + {getFSSourceAccessor(), CanonPath(tmpDir.string())}, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256, + {}, + filter); + }() + : [&]() { + // Nix < 2.20 used `git archive` for repos without submodules. + auto source = sinkToSource([&](Sink & sink) { + runProgram2( + {.program = "git", + .args = {"-C", repoDir, "--git-dir", repoInfo.gitDir, "archive", rev.gitRev()}, + .standardOut = &sink}); + }); + + unpackTarfile(*source, tmpDir); + + return store.addToStore("source", {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + }(); + + settings.getCache()->upsert( + cacheKey, {{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}}); + + return makeAccessor(storePath); + } + + std::optional, Input>> getAccessorFromCommit( + const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input, bool fastOnly) const { assert(!repoInfo.workdirInfo.isDirty); auto origRev = input.getRev(); auto originalRef = input.getRef(); - bool shallow = getShallowAttr(input); + bool shallow = canDoShallow(input); auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow); input.attrs.insert_or_assign("ref", ref); @@ -816,12 +940,28 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir, {})->resolveRef(ref).gitRev()); } else { + auto rev = input.getRev(); auto repoUrl = std::get(repoInfo.location); std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow); repoDir = cacheDir; repoInfo.gitDir = "."; - std::filesystem::create_directories(cacheDir.parent_path()); + /* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that + * repo instead. */ + std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false); + if (rev && shallow && pathExists(cacheDirNonShallow)) { + auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, {.create = true, .bare = true}); + if (nonShallowRepo->hasObject(*rev)) { + debug( + "using non-shallow cached repo for '%s' since it contains rev '%s'", + repoUrl.to_string(), + rev->gitRev()); + repoDir = cacheDirNonShallow; + goto have_rev; + } + } + + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); @@ -836,7 +976,7 @@ struct GitInputScheme : InputScheme /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (auto rev = input.getRev()) { + if (rev) { doFetch = !repo->hasObject(*rev); } else { if (getAllRefsAttr(input)) { @@ -850,7 +990,9 @@ struct GitInputScheme : InputScheme } if (doFetch) { - bool shallow = getShallowAttr(input); + if (fastOnly) + return std::nullopt; + try { auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*" : input.getRev() ? input.getRev()->gitRev() @@ -878,7 +1020,7 @@ struct GitInputScheme : InputScheme warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); } - if (auto rev = input.getRev()) { + if (rev) { if (!repo->hasObject(*rev)) throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " @@ -895,40 +1037,88 @@ struct GitInputScheme : InputScheme // the remainder } + have_rev: auto repo = GitRepo::openRepo(repoDir, {}); - auto isShallow = repo->isShallow(); - - if (isShallow && !getShallowAttr(input)) - throw Error( - "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", - repoInfo.locationToArg()); - // FIXME: check whether rev is an ancestor of ref? auto rev = *input.getRev(); - input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); + /* Skip lastModified computation if it's already supplied by the caller. + We don't care if they specify an incorrect value; it doesn't + matter for security, unlike narHash. */ + if (!input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); - if (!getShallowAttr(input)) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + /* Like lastModified, skip revCount if supplied by the caller. */ + if (!shallow && !input.attrs.contains("revCount")) { + auto isShallow = repo->isShallow(); + + if (isShallow && !shallow) + throw Error( + "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", + repoInfo.locationToArg()); + + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); + } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); verifyCommit(input, repo); - bool exportIgnore = getExportIgnoreAttr(input); - bool smudgeLfs = getLfsAttr(input); - auto accessor = repo->getAccessor( - rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»"); + auto options = getGitAccessorOptions(input); + + auto expectedNarHash = input.getNarHash(); + + auto accessor = repo->getAccessor(rev, options, "«" + input.to_string(true) + "»"); + + if (settings.nix219Compat && !options.smudgeLfs) { + /* Use Nix 2.19 semantics to generate locks, but if a NAR hash is specified, support Nix >= 2.20 semantics + * as well. */ + warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string()); + auto accessorModern = accessor; + accessor = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); + if (expectedNarHash) { + auto narHashLegacy = + fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; + if (expectedNarHash != narHashLegacy) { + auto narHashModern = + fetchToStore2(settings, store, {accessorModern}, FetchMode::DryRun, input.getName()).second; + if (expectedNarHash == narHashModern) + accessor = accessorModern; + } + } + } else { + /* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Nix applying Git filters, + * `export-ignore` or `export-subst`. Nix >= 2.20 doesn't do those, so we may get a NAR hash mismatch. If + * that happens, try again using `git archive`. */ + if (expectedNarHash) { + auto narHashNew = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; + if (expectedNarHash != narHashNew) { + auto accessorLegacy = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); + auto narHashLegacy = + fetchToStore2(settings, store, {accessorLegacy}, FetchMode::DryRun, input.getName()).second; + if (expectedNarHash == narHashLegacy) { + warn( + "Git input '%s' specifies a NAR hash '%s' that was created by Nix < 2.20.\n" + "Nix >= 2.20 does not apply Git filters, `export-ignore` and `export-subst` by default, which changes the NAR hash.\n" + "Please update the NAR hash to '%s'.", + input.to_string(), + expectedNarHash->to_string(HashFormat::SRI, true), + narHashNew.to_string(HashFormat::SRI, true)); + accessor = accessorLegacy; + } + } + } + } /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodules. */ - if (getSubmodulesAttr(input)) { + if (options.submodules) { std::map> mounts; - for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options.exportIgnore)) { auto resolved = repo->resolveSubmoduleUrl(submodule.url); debug( "Git submodule %s: %s %s %s -> %s", @@ -951,25 +1141,27 @@ struct GitInputScheme : InputScheme } } attrs.insert_or_assign("rev", submoduleRev.gitRev()); - attrs.insert_or_assign("exportIgnore", Explicit{exportIgnore}); + attrs.insert_or_assign("exportIgnore", Explicit{options.exportIgnore}); attrs.insert_or_assign("submodules", Explicit{true}); - attrs.insert_or_assign("lfs", Explicit{smudgeLfs}); + attrs.insert_or_assign("lfs", Explicit{options.smudgeLfs}); attrs.insert_or_assign("allRefs", Explicit{true}); auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); mounts.insert_or_assign(submodule.path, submoduleAccessor); } if (!mounts.empty()) { + auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s"); mounts.insert_or_assign(CanonPath::root, accessor); accessor = makeMountedSourceAccessor(std::move(mounts)); + accessor->fingerprint = newFingerprint; } } assert(!origRev || origRev == rev); - return {accessor, std::move(input)}; + return {{accessor, std::move(input)}}; } std::pair, Input> @@ -1007,7 +1199,7 @@ struct GitInputScheme : InputScheme auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); /* If the submodule is dirty, mark this repo dirty as well. */ @@ -1032,8 +1224,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); @@ -1056,8 +1251,8 @@ struct GitInputScheme : InputScheme return {accessor, std::move(input)}; } - std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + std::optional, Input>> + getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override { Input input(_input); @@ -1072,22 +1267,19 @@ struct GitInputScheme : InputScheme throw UnimplementedError("exportIgnore and submodules are not supported together yet"); } - auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath() - ? getAccessorFromCommit(settings, store, repoInfo, std::move(input)) - : getAccessorFromWorkdir(settings, store, repoInfo, std::move(input)); - - return {accessor, std::move(final)}; + return input.getRef() || input.getRev() || !repoInfo.getPath() + ? getAccessorFromCommit(settings, store, repoInfo, std::move(input), fastOnly) + : std::optional{getAccessorFromWorkdir(settings, store, repoInfo, std::move(input))}; } std::optional getFingerprint(Store & store, const Input & input) const override { - auto makeFingerprint = [&](const Hash & rev) { - return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") - + (getLfsAttr(input) ? ";l" : ""); - }; + auto options = getGitAccessorOptions(input); if (auto rev = input.getRev()) - return makeFingerprint(*rev); + // FIXME: this can return a wrong fingerprint for the legacy (`git archive`) case, since we don't know here + // whether to append the `;legacy` suffix or not. + return options.makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { @@ -1097,13 +1289,13 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + return options.makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false); } return std::nullopt; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b86fa926a668..930aeb050b5e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,6 +1,5 @@ #include "nix/store/filetransfer.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/url-parts.hh" @@ -13,7 +12,6 @@ #include #include -#include namespace nix::fetchers { @@ -41,16 +39,14 @@ struct GitArchiveInputScheme : InputScheme /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); - std::optional rev; - std::optional ref; - std::optional host_url; + Attrs attrs; auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = path[2]; + attrs.insert_or_assign("rev", path[2]); else - ref = path[2]; + attrs.insert_or_assign("ref", path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -59,38 +55,30 @@ struct GitArchiveInputScheme : InputScheme rs += "/"; } } - ref = rs; + attrs.insert_or_assign("ref", rs); } else if (size < 2) throw BadURL("URL '%s' is invalid", url); for (auto & [name, value] : url.query) { if (name == "rev") { - if (rev) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple commit hashes", url); - rev = value; + attrs.insert_or_assign("rev", value); } else if (name == "ref") { - if (ref) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple branch/tag names", url); - ref = value; + attrs.insert_or_assign("ref", value); } else if (name == "host") - host_url = value; - // FIXME: barf on unsupported attributes + attrs.insert_or_assign("host", value); + else if (name == "narHash") + attrs.insert_or_assign("narHash", value); + else + throw BadURL("URL '%s' contains unknown parameter '%s'", url, name); } - Attrs attrs; attrs.insert_or_assign("type", std::string{schemeName()}); attrs.insert_or_assign("owner", path[0]); attrs.insert_or_assign("repo", path[1]); - if (rev) - attrs.insert_or_assign("rev", *rev); - if (ref) - attrs.insert_or_assign("ref", *ref); - if (host_url) - attrs.insert_or_assign("host", *host_url); - - auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) - attrs.insert_or_assign("narHash", narHash->second); return inputFromAttrs(settings, attrs); } @@ -162,7 +150,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); @@ -173,7 +161,7 @@ struct GitArchiveInputScheme : InputScheme if (ref) path.push_back(*ref); if (rev) - path.push_back(rev->to_string(HashFormat::Base16, false)); + path.push_back(abbreviate ? rev->gitShortRev() : rev->gitRev()); auto url = ParsedURL{ .scheme = std::string{schemeName()}, .path = path, @@ -271,7 +259,8 @@ struct GitArchiveInputScheme : InputScheme time_t lastModified; }; - std::pair downloadArchive(const Settings & settings, Store & store, Input input) const + std::optional> + downloadArchive(const Settings & settings, Store & store, Input input, bool fastOnly) const { if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); @@ -299,12 +288,16 @@ struct GitArchiveInputScheme : InputScheme auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); if (settings.getTarballCache()->hasObject(treeHash)) - return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}; + return { + {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); } } + if (fastOnly) + return std::nullopt; + /* Stream the tarball into the tarball cache. */ auto url = getDownloadUrl(settings, input); @@ -340,13 +333,17 @@ struct GitArchiveInputScheme : InputScheme rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev()); #endif - return {std::move(input), tarballInfo}; + return {{std::move(input), tarballInfo}}; } - std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + std::optional, Input>> + getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override { - auto [input, tarballInfo] = downloadArchive(settings, store, _input); + auto res = downloadArchive(settings, store, _input, fastOnly); + if (fastOnly && !res) + return std::nullopt; + assert(res); + auto [input, tarballInfo] = *res; #if 0 input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); @@ -354,9 +351,16 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); auto accessor = - settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»"); + settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string(true) + "»"); + + if (!settings.trustTarballsFromGitForges) + // FIXME: computing the NAR hash here is wasteful if + // copyInputToStore() is just going to hash/copy it as + // well. + input.attrs.insert_or_assign( + "narHash", accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true)); - return {accessor, input}; + return {{accessor, input}}; } bool isLocked(const Settings & settings, const Input & input) const override @@ -368,15 +372,10 @@ struct GitArchiveInputScheme : InputScheme return input.getRev().has_value() && (settings.trustTarballsFromGitForges || input.getNarHash().has_value()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - std::optional getFingerprint(Store & store, const Input & input) const override { if (auto rev = input.getRev()) - return rev->gitRev(); + return "github:" + rev->gitRev(); else return std::nullopt; } @@ -454,8 +453,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : headers.empty() ? "https://%s/%s/%s/archive/%s.tar.gz" : "https://api.%s/repos/%s/%s/tarball/%s"; - const auto url = - fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(HashFormat::Base16, false)); + const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->gitRev()); return DownloadUrl{parseURL(url), headers}; } @@ -542,7 +540,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(HashFormat::Base16, false)); + input.getRev()->gitRev()); Headers headers = makeHeadersWithAuthTokens(settings, host, input); return DownloadUrl{parseURL(url), headers}; @@ -638,7 +636,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(HashFormat::Base16, false)); + input.getRev()->gitRev()); Headers headers = makeHeadersWithAuthTokens(settings, host, input); return DownloadUrl{parseURL(url), headers}; diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf69..8eede58086e5 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,16 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 7219635ec07d..fd21d7f384d3 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -12,6 +12,11 @@ namespace nix::fetchers { */ struct Cache { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: virtual ~Cache() {} /** @@ -67,9 +72,9 @@ struct Cache /** * Look up a store path in the cache. The returned store path will - * be valid, but it may be expired. + * be valid (unless `allowInvalid` is true), but it may be expired. */ - virtual std::optional lookupStorePath(Key key, Store & store) = 0; + virtual std::optional lookupStorePath(Key key, Store & store, bool allowInvalid = false) = 0; /** * Look up a store path in the cache. Return nothing if its TTL diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 2ab215a685ff..b68e2c0316f6 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -14,8 +14,9 @@ namespace nix { struct GitRepo; +struct SrcToStore; -} +} // namespace nix namespace nix::fetchers { @@ -94,10 +95,7 @@ struct Settings : public Config are subsequently modified. Therefore lock files with dirty locks should generally only be used for local testing, and should not be pushed to other users. - )", - {}, - true, - Xp::Flakes}; + )"}; Setting trustTarballsFromGitForges{ this, @@ -118,16 +116,23 @@ struct Settings : public Config Setting flakeRegistry{ this, - "https://channels.nixos.org/flake-registry.json", + "https://install.determinate.systems/flake-registry/stable/flake-registry.json", "flake-registry", R"( Path or URI of the global flake registry. When empty, disables the global flake registry. - )", - {}, - true, - Xp::Flakes}; + )"}; + + Setting nix219Compat{ + this, + false, + "nix-219-compat", + R"( + If enabled, Nix will generate lock files that are compatible with Nix 2.19. + In particular, Nix will use `git archive` rather than `libgit2` to copy Git inputs. + The resulting locks may not be compatible with Nix >= 2.20. + )"}; Setting tarballTtl{ this, @@ -152,8 +157,30 @@ struct Settings : public Config ref getTarballCache() const; + /** + * In-memory cache for calls to fetchToStore(); maps source paths to their store + * paths / hashes. + */ + static ref createSrcToStore(); + + const ref srcToStore = createSrcToStore(); + + private: + void anchor() override; + mutable Sync> _cache; + + mutable Sync> _tarballCache; }; } // namespace nix::fetchers + +namespace nix { + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +} // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 180d10e9dbbf..d830d83c840a 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -61,11 +61,11 @@ public: */ static Input fromAttrs(const Settings & settings, Attrs && attrs); - ParsedURL toURL() const; + ParsedURL toURL(bool abbreviate = false) const; - std::string toURLString(const StringMap & extraQuery = {}) const; + std::string toURLString(const StringMap & extraQuery = {}, bool abbreviate = false) const; - std::string to_string() const; + std::string to_string(bool abbreviate = false) const; Attrs toAttrs() const; @@ -113,7 +113,7 @@ public: * Fetch the entire input into the Nix store, returning the * location in the Nix store and the locked input. */ - std::pair fetchToStore(const Settings & settings, Store & store) const; + std::tuple, Input> fetchToStore(const Settings & settings, Store & store) const; /** * Check the locking attributes in `result` against @@ -225,7 +225,7 @@ struct InputScheme */ virtual const std::map & allowedAttrs() const = 0; - virtual ParsedURL toURL(const Input & input) const; + virtual ParsedURL toURL(const Input & input, bool abbreviate = false) const; virtual Input applyOverrides(const Input & input, std::optional ref, std::optional rev) const; @@ -240,8 +240,19 @@ struct InputScheme std::string_view contents, std::optional commitMsg) const; + virtual std::optional, Input>> + getAccessor(const Settings & settings, Store & store, const Input & input, bool fastOnly) const + { + if (fastOnly) + return std::nullopt; + return getAccessor(settings, store, input); + } + virtual std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & input) const = 0; + getAccessor(const Settings & settings, Store & store, const Input & input) const + { + return getAccessor(settings, store, input, false).value(); + } /** * Is this `InputScheme` part of an experimental feature? diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 98532c4b14c2..6b7a9d83ad10 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,8 @@ #include "nix/util/source-path.hh" -#include +#include +#include namespace nix { @@ -52,7 +53,12 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; + std::shared_ptr getProvenance(const CanonPath & path) override; + + void invalidateCache() override + { + next->invalidateCache(); + } /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` @@ -79,8 +85,8 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh index b59da391a056..444e0a648f14 100644 --- a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh +++ b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh @@ -38,9 +38,17 @@ struct Fetch void fetch( const std::string & content, const CanonPath & pointerFilePath, - StringSink & sink, + Sink & sink, std::function sizeCallback) const; std::vector fetchUrls(const std::vector & pointers) const; }; +struct LfsApiInfo +{ + std::string endpoint; + std::optional authHeader; +}; + +LfsApiInfo getLfsApi(ParsedURL url); + } // namespace nix::lfs diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 24a7b80087f8..6ffc7372df0f 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -16,6 +16,10 @@ struct Settings; */ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { +private: + void anchor() override; + +public: /** * Flush builder and return a final Git hash. */ @@ -26,6 +30,9 @@ struct GitAccessorOptions { bool exportIgnore = false; bool smudgeLfs = false; + bool submodules = false; // Currently implemented in GitInputScheme rather than GitAccessor + + std::string makeFingerprint(const Hash & rev) const; }; struct GitRepo @@ -88,6 +95,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; @@ -133,17 +143,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -// A helper to ensure that the `git_*_free` functions get called. -template -struct Deleter -{ - template - void operator()(T * p) const - { - del(p); - }; -}; - // A helper to ensure that we don't leak objects returned by libgit2. template struct Setter diff --git a/src/libfetchers/include/nix/fetchers/meson.build b/src/libfetchers/include/nix/fetchers/meson.build index a313b1e0bc0c..f3bb80942a28 100644 --- a/src/libfetchers/include/nix/fetchers/meson.build +++ b/src/libfetchers/include/nix/fetchers/meson.build @@ -10,6 +10,7 @@ headers = files( 'git-lfs-fetch.hh', 'git-utils.hh', 'input-cache.hh', + 'provenance.hh', 'registry.hh', 'tarball.hh', ) diff --git a/src/libfetchers/include/nix/fetchers/provenance.hh b/src/libfetchers/include/nix/fetchers/provenance.hh new file mode 100644 index 000000000000..82dc1b3b8109 --- /dev/null +++ b/src/libfetchers/include/nix/fetchers/provenance.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "nix/util/provenance.hh" +#include "nix/fetchers/fetchers.hh" + +namespace nix { + +struct TreeProvenance : Provenance +{ + ref attrs; + + TreeProvenance(const fetchers::Input & input); + + TreeProvenance(ref attrs) + : attrs(std::move(attrs)) + { + } + + nlohmann::json to_json() const override; +}; + +struct FetchurlProvenance : Provenance +{ + std::string url; + + FetchurlProvenance(std::string url, bool sanitize = true); + + nlohmann::json to_json() const override; +}; + +} // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/registry.hh b/src/libfetchers/include/nix/fetchers/registry.hh index dc7e3edb590e..ca38dd805d6d 100644 --- a/src/libfetchers/include/nix/fetchers/registry.hh +++ b/src/libfetchers/include/nix/fetchers/registry.hh @@ -39,6 +39,9 @@ struct Registry static std::shared_ptr read(const Settings & settings, const SourcePath & path, RegistryType type); + static std::shared_ptr + read(const Settings & settings, std::string_view whence, std::string_view jsonStr, RegistryType type); + void write(const std::filesystem::path & path); void add(const Input & from, const Input & to, const Attrs & extraAttrs); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b2a41a7421fd..e629dcbac6b0 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -100,7 +100,7 @@ struct IndirectInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { ParsedURL url{ .scheme = "flake", @@ -131,11 +131,6 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - bool isDirect(const Input & input) const override { return false; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 652d5ce7976b..3fe96d8503bc 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,7 +1,7 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" -#include "nix/util/source-path.hh" namespace nix::fetchers { @@ -66,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 38fe31fcadea..b7023efab609 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -7,8 +7,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" #include "nix/fetchers/fetch-settings.hh" @@ -30,10 +28,9 @@ static RunOptions hgOptions(OsStrings args) } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -static std::string runHg(OsStrings args, const std::optional & input = {}) +static std::string runHg(OsStrings args) { RunOptions opts = hgOptions(std::move(args)); - opts.input = input; auto res = runProgram(std::move(opts)); @@ -125,7 +122,7 @@ struct MercurialInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); url.scheme = "hg+" + url.scheme; @@ -243,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -282,9 +278,7 @@ struct MercurialInputScheme : InputScheme auto revInfoKey = [&](const Hash & rev) { if (rev.algo != HashAlgorithm::SHA1) - throw Error( - "Hash '%s' is not supported by Mercurial. Only sha1 is supported.", - rev.to_string(HashFormat::Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.gitRev()); return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}}; }; @@ -384,7 +378,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, @@ -406,7 +400,7 @@ struct MercurialInputScheme : InputScheme auto storePath = fetchToStore(settings, store, input); auto accessor = store.requireStoreObjectAccessor(storePath); - accessor->setPathDisplay("«" + input.to_string() + "»"); + accessor->setPathDisplay("«" + input.to_string(true) + "»"); return {accessor, input}; } @@ -419,7 +413,7 @@ struct MercurialInputScheme : InputScheme std::optional getFingerprint(Store & store, const Input & input) const override { if (auto rev = input.getRev()) - return rev->gitRev(); + return "hg:" + rev->gitRev(); else return std::nullopt; } diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d34dd4f434d1..134fd496a7a1 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -35,6 +35,7 @@ subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', + 'builtin.cc', 'cache.cc', 'fetch-settings.cc', 'fetch-to-store.cc', @@ -48,23 +49,30 @@ sources = files( 'input-cache.cc', 'mercurial.cc', 'path.cc', + 'provenance.cc', 'registry.cc', 'tarball.cc', ) subdir('include/nix/fetchers') +# Generate builtin-flake-registry.json.gen.hh +subdir('nix-meson-build-support/generate-header') + +sources += gen_header.process( + 'builtin-flake-registry.json', +) + subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') this_library = library( 'nixfetchers', sources, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 14592087999c..1a30ac293018 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-fetchers"; + pname = "determinate-nix-fetchers"; inherit version; workDir = ./.; @@ -28,6 +28,7 @@ mkMesonLibrary (finalAttrs: { ./.version ./meson.build ./include/nix/fetchers/meson.build + ./builtin-flake-registry.json (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index cf7f6aa920dd..9844c7a84b31 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -87,7 +87,7 @@ struct PathInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { auto query = attrsToQuery(input.attrs); query.erase("path"); @@ -95,7 +95,7 @@ struct PathInputScheme : InputScheme query.erase("__final"); return ParsedURL{ .scheme = "path", - .path = splitString>(getStrAttr(input.attrs, "path"), "/"), + .path = pathToUrlPath(std::filesystem::path{getStrAttr(input.attrs, "path")}), .query = query, }; } @@ -138,51 +138,37 @@ struct PathInputScheme : InputScheme throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } - std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + std::optional, Input>> + getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override { + // Note: fastOnly is ignored because the path fetcher is always fast. + Input input(_input); - auto path = getStrAttr(input.attrs, "path"); auto absPath = getAbsPath(input); // FIXME: check whether access to 'path' is allowed. + + auto accessor = makeFSSourceAccessor(absPath); + auto storePath = store.maybeParseStorePath(absPath.string()); - if (storePath) + if (storePath) { store.addTempRoot(*storePath); - time_t mtime = 0; - if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", PathFmt(absPath))); - // FIXME: try to substitute storePath. - auto src = sinkToSource( - [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); - storePath = store.addToStoreFromDump(*src, "source"); + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store.maybeQueryPathInfo(*storePath); + if (info) { + accessor->fingerprint = fmt("path:%s", info->narHash.to_string(HashFormat::SRI, true)); + settings.getCache()->upsert( + makeSourcePathToHashCacheKey( + *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), + {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); + } } - auto accessor = store.requireStoreObjectAccessor(*storePath); - - // To prevent `fetchToStore()` copying the path again to Nix - // store, pre-create an entry in the fetcher cache. - auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true); - accessor->fingerprint = fmt("path:%s", narHash); - settings.getCache()->upsert( - makeSourcePathToHashCacheKey( - *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), - {{"hash", narHash}}); - - /* Trust the lastModified value supplied by the user, if - any. It's not a "secure" attribute so we don't care. */ - if (!input.getLastModified()) - input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - - return {accessor, std::move(input)}; - } - - std::optional experimentalFeature() const override - { - return Xp::Flakes; + return {{accessor, std::move(input)}}; } }; diff --git a/src/libfetchers/provenance.cc b/src/libfetchers/provenance.cc new file mode 100644 index 000000000000..e984b2271014 --- /dev/null +++ b/src/libfetchers/provenance.cc @@ -0,0 +1,58 @@ +#include "nix/fetchers/provenance.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/util/json-utils.hh" + +#include + +namespace nix { + +TreeProvenance::TreeProvenance(const fetchers::Input & input) + : attrs(make_ref([&]() { + // Remove the narHash attribute from the provenance info, as it's redundant (it's already recorded in the store + // path info). + auto attrs2 = input.attrs; + attrs2.erase("narHash"); + return fetchers::attrsToJSON(attrs2); + }())) +{ +} + +nlohmann::json TreeProvenance::to_json() const +{ + return nlohmann::json{ + {"type", "tree"}, + {"attrs", *attrs}, + }; +} + +Provenance::Register registerTreeProvenance("tree", [](nlohmann::json json) { + auto & obj = getObject(json); + auto & attrsJson = valueAt(obj, "attrs"); + return make_ref(make_ref(attrsJson)); +}); + +FetchurlProvenance::FetchurlProvenance(std::string _url, bool sanitize) + : url(std::move(_url)) +{ + if (sanitize) { + try { + url = parseURL(url, true).renderSanitized(); + } catch (BadURL &) { + } + } +} + +nlohmann::json FetchurlProvenance::to_json() const +{ + return nlohmann::json{ + {"type", "fetchurl"}, + {"url", url}, + }; +} + +Provenance::Register registerFetchurlProvenance("fetchurl", [](nlohmann::json json) { + auto & obj = getObject(json); + return make_ref(getString(valueAt(obj, "url")), false); +}); + +} // namespace nix diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 9911586fa037..82a65061deae 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,8 +1,8 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/registry.hh" #include "nix/fetchers/tarball.hh" +#include "nix/store/filetransfer.hh" #include "nix/util/users.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" @@ -14,14 +14,24 @@ std::shared_ptr Registry::read(const Settings & settings, const Source { debug("reading registry '%s'", path); - auto registry = std::make_shared(type); - if (!path.pathExists()) return std::make_shared(type); try { + return read(settings, path.to_string(), path.readFile(), type); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); + return std::make_shared(type); + } +} - auto json = nlohmann::json::parse(path.readFile()); +std::shared_ptr +Registry::read(const Settings & settings, std::string_view whence, std::string_view jsonStr, RegistryType type) +{ + auto registry = std::make_shared(type); + + try { + auto json = nlohmann::json::parse(jsonStr); auto version = json.value("version", 0); @@ -45,12 +55,10 @@ std::shared_ptr Registry::read(const Settings & settings, const Source } else - throw Error("flake registry '%s' has unsupported version %d", path, version); + warn("flake registry '%s' has unsupported version %d", whence, version); } catch (nlohmann::json::exception & e) { - warn("cannot parse flake registry '%s': %s", path, e.what()); - } catch (Error & e) { - warn("cannot read flake registry '%s': %s", path, e.what()); + warn("cannot parse flake registry '%s': %s", whence, e.what()); } return registry; @@ -139,25 +147,39 @@ void overrideRegistry(const Input & from, const Input & to, const Attrs & extraA static std::shared_ptr getGlobalRegistry(const Settings & settings, Store & store) { static auto reg = [&]() { - auto path = settings.flakeRegistry.get(); - if (path == "") { - return std::make_shared(Registry::Global); // empty registry - } + try { + auto path = settings.flakeRegistry.get(); + if (path == "") { + return std::make_shared(Registry::Global); // empty registry + } - return Registry::read( - settings, - [&] -> SourcePath { - std::filesystem::path fsPath{path}; - if (!fsPath.is_absolute()) { - auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; - if (auto store2 = dynamic_cast(&store)) - store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string()); - return {store.requireStoreObjectAccessor(storePath)}; - } else { - return SourcePath{getFSSourceAccessor(), CanonPath{fsPath.string()}}.resolveSymlinks(); - } - }(), - Registry::Global); + return Registry::read( + settings, + [&] -> SourcePath { + std::filesystem::path fsPath{path}; + if (!fsPath.is_absolute()) { + auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; + if (auto store2 = dynamic_cast(&store)) + store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string()); + return {store.requireStoreObjectAccessor(storePath)}; + } else { + return SourcePath{getFSSourceAccessor(), CanonPath{fsPath.string()}}.resolveSymlinks(); + } + }(), + Registry::Global); + } catch (Error & e) { + warn( + "cannot fetch global flake registry '%s', will use builtin fallback registry: %s", + settings.flakeRegistry.get(), + e.info().msg); + // Use builtin registry as fallback + return Registry::read( + settings, + "builtin flake registry", +#include "builtin-flake-registry.json.gen.hh" + , + Registry::Global); + } }(); return reg; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 065174433110..5586229e56cf 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,6 +9,9 @@ #include "nix/store/store-api.hh" #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/fetch-settings.hh" +#include "nix/fetchers/provenance.hh" + +#include namespace nix::fetchers { @@ -83,6 +86,8 @@ DownloadFileResult downloadFile( }, hashString(HashAlgorithm::SHA256, sink.s)); info.narSize = sink.s.size(); + if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) + info.provenance = std::make_shared(request.uri.to_string()); auto source = StringSource{sink.s}; store.addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); @@ -104,8 +109,12 @@ DownloadFileResult downloadFile( }; } -static DownloadTarballResult downloadTarball_( - const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix) +static std::optional downloadTarball_( + const Settings & settings, + const std::string & urlS, + const Headers & headers, + const std::string & displayPrefix, + bool fastOnly) { ParsedURL url = parseURL(urlS); @@ -155,6 +164,9 @@ static DownloadTarballResult downloadTarball_( `tarballTtl`, so no need to check the server. */ return attrsToResult(cached->value); + if (fastOnly) + return std::nullopt; + auto _res = std::make_shared>(); auto source = sinkToSource([&](Sink & sink) { @@ -382,7 +394,7 @@ struct CurlInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) const override + ParsedURL toURL(const Input & input, bool abbreviate) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); // NAR hashes are preferred over file hashes since tar/zip @@ -437,7 +449,7 @@ struct FileInputScheme : CurlInputScheme auto accessor = ref{store.getFSAccessor(file.storePath)}; - accessor->setPathDisplay("«" + input.to_string() + "»"); + accessor->setPathDisplay("«" + input.to_string(true) + "»"); return {accessor, input}; } @@ -487,12 +499,16 @@ struct TarballInputScheme : CurlInputScheme : (requireTree || hasTarballExtension(url))); } - std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + std::optional, Input>> + getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override { auto input(_input); - auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); + auto res = + downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string(true) + "»", fastOnly); + if (!res) + return std::nullopt; + auto & result = *res; if (result.immutableUrl) { auto immutableInput = Input::fromURL(settings, *result.immutableUrl); @@ -510,15 +526,15 @@ struct TarballInputScheme : CurlInputScheme "narHash", settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true)); - return {result.accessor, input}; + return {{result.accessor, input}}; } std::optional getFingerprint(Store & store, const Input & input) const override { if (auto narHash = input.getNarHash()) - return narHash->to_string(HashFormat::SRI, true); + return "tarball:" + narHash->to_string(HashFormat::SRI, true); else if (auto rev = input.getRev()) - return rev->gitRev(); + return "tarball:" + rev->gitRev(); else return std::nullopt; } diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fddb39bdf96b..b1067a5d054e 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,16 +50,32 @@ headers += files('nix_api_flake.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixflakec', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixflakec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixflakec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libflake-c/meson.options b/src/libflake-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libflake-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index cbf5430f54a9..2558236a7e5a 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -209,4 +209,20 @@ nix_value * nix_locked_flake_get_output_attrs( NIXC_CATCH_ERRS_NULL } +nix_err nix_locked_flake_read_path( + nix_c_context * context, + nix_locked_flake * lockedFlake, + const char * path, + nix_get_string_callback callback, + void * user_data) +{ + nix_clear_err(context); + try { + auto source_path = lockedFlake->lockedFlake->flake.path.parent() / nix::CanonPath(path); + auto v = source_path.readFile(); + return call_nix_get_string_callback(v, callback, user_data); + } + NIXC_CATCH_ERRS +} + } // extern "C" diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h index a3221e676bec..9884a3d39a49 100644 --- a/src/libflake-c/nix_api_flake.h +++ b/src/libflake-c/nix_api_flake.h @@ -239,6 +239,23 @@ void nix_flake_reference_free(nix_flake_reference * store); nix_value * nix_locked_flake_get_output_attrs( nix_c_context * context, nix_flake_settings * settings, EvalState * evalState, nix_locked_flake * lockedFlake); +/** + * @brief Reads a file within the flake. + * @note The callback borrows the string only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] locked_flake the flake to get the path for + * @param[in] path The path within the flake. + * @param[in] callback The callback to call with the string + * @param[in] user_data Additional data to pass for the callback + */ +nix_err nix_locked_flake_read_path( + nix_c_context * context, + nix_locked_flake * lockedFlake, + const char * path, + nix_get_string_callback callback, + void * user_data); + #ifdef __cplusplus } // extern "C" #endif diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 8c6883d9cf95..344412b47fc0 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -17,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake-c"; + pname = "determinate-nix-flake-c"; inherit version; workDir = ./.; @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 5fcbbc9c30bf..332ffe18b7dc 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -17,8 +17,6 @@ namespace nix { TEST(parseFlakeRef, path) { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - fetchers::Settings fetchSettings; { @@ -67,8 +65,6 @@ TEST(parseFlakeRef, path) TEST(parseFlakeRef, GitArchiveInput) { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - fetchers::Settings fetchSettings; { @@ -111,7 +107,6 @@ class InputFromURLTest : public ::testing::WithParamInterface= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -60,8 +60,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'NIX_CONFIG' : 'extra-experimental-features = flakes', 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 397ef4192447..2a359e1c2b47 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -60,7 +60,6 @@ mkMesonExecutable (finalAttrs: { } ('' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - export NIX_CONFIG="extra-experimental-features = flakes" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index ed7947e06012..d1037efcd23b 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -10,9 +10,6 @@ lockFileStr: # unlocked trees. overrides: -# This is `prim_fetchFinalTree`. -fetchTreeFinal: - let inherit (builtins) mapAttrs; @@ -45,14 +42,24 @@ let parentNode = allNodes.${getInputByPath lockFile.root node.parent}; sourceInfo = - if hasOverride then + if node.buildTime or false then + derivation { + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = node.locked; + outputHashMode = "recursive"; + outputHash = node.locked.narHash; + } + else if hasOverride then overrides.${key}.sourceInfo else if isRelative then parentNode.sourceInfo else # FIXME: remove obsolete node.info. # Note: lock file entries are always final. - fetchTreeFinal (node.info or { } // removeAttrs node.locked [ "dir" ]); + builtins.fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]); subdir = overrides.${key}.dir or node.locked.dir or ""; @@ -93,6 +100,7 @@ let result = if node.flake or true then assert builtins.isFunction flake.outputs; + assert !(node.buildTime or false); result else sourceInfo // { inherit sourceInfo outPath; }; diff --git a/src/libflake/config.cc b/src/libflake/config.cc index b7d5cd8999e5..6367d7b4ec42 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -11,14 +11,12 @@ #include #include #include -#include #include "nix/util/users.hh" #include "nix/util/config-global.hh" #include "nix/flake/settings.hh" #include "nix/flake/flake.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/file-system.hh" #include "nix/util/fmt.hh" #include "nix/util/logging.hh" diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 4b84c081e65f..b8838ab18e52 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -1,14 +1,11 @@ #include -#include #include #include -#include #include -#include #include -#include #include "nix/flake/flake-primops.hh" +#include "nix/store/store-api.hh" #include "nix/expr/eval.hh" #include "nix/flake/flake.hh" #include "nix/flake/flakeref.hh" @@ -21,49 +18,75 @@ #include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" +#include "nix/util/mounted-source-accessor.hh" namespace nix::flake::primops { PrimOp getFlake(const Settings & settings) { auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value ** args, Value & v) { - std::string flakeRefS( - state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) - throw Error( - "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", - flakeRefS, - state.positions[pos]); - - callFlake( - state, - lockFlake( - settings, - state, - flakeRef, - LockFlags{ - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && settings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); + state.forceValue(*args[0], pos); + + LockFlags lockFlags{ + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }; + + if (args[0]->type() == nPath) { + auto path = state.realisePath(pos, *args[0]); + callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } else { + NixStringContext context; + std::string flakeRefS( + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getFlake")); + auto rewrites = state.realiseContext(context); + flakeRefS = state.devirtualize(rewriteStrings(flakeRefS, rewrites), context); + if (hasContext(context)) + // FIXME: this should really be an error. + warn( + "In 'builtins.getFlake', the flakeref '%s' has string context, but that's not allowed. This may become a fatal error in the future.", + flakeRefS); + + auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) + throw Error( + "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", + flakeRefS, + state.positions[pos]); + + /* Backward compatibility hack: If this is a `path` flake and it's a virtual path that had + * `unsafeDiscardStringContext` applied to it, then treat it like the `nPath` case, i.e. call lockFlake() on + * the virtual path directly. This is necessary because the `path` fetcher doesn't see virtual paths. */ + if (flakeRef.input.getType() == "path") { + if (auto sourcePath = flakeRef.input.getSourcePath(); + sourcePath && state.store->isInStore(sourcePath->string())) { + auto [storePath, subPath] = state.store->toStorePath(sourcePath->string()); + if (auto mount = state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + auto path = state.storePath(storePath) / CanonPath(subPath); + if (!flakeRef.subdir.empty()) + path = path / flakeRef.subdir; + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } + } + } + + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); + } }; return PrimOp{ .name = "__getFlake", .args = {"args"}, .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + Fetch a flake from a flake reference or a path, and return its output attributes and some metadata. For example: ```nix (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix @@ -78,7 +101,6 @@ PrimOp getFlake(const Settings & settings) ``` )", .impl = prim_getFlake, - .experimentalFeature = Xp::Flakes, }; } @@ -91,12 +113,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } @@ -120,13 +143,13 @@ nix::PrimOp parseFlakeRef({ ``` )", .impl = prim_parseFlakeRef, - .experimentalFeature = Xp::Flakes, }); static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** args, Value & v) { state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString"); fetchers::Attrs attrs; + NixStringContext context; for (const auto & attr : *args[0]->attrs()) { state.forceValue(*attr.value, attr.pos); auto t = attr.value->type(); @@ -145,7 +168,9 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); } else if (t == nString) { - attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view())); + auto s = state.forceString( + *attr.value, context, attr.pos, "while evaluating an attribute in 'builtins.flakeRefToString'"); + attrs.emplace(state.symbols[attr.name], std::string(s)); } else { state .error( @@ -157,7 +182,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** } } auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); - v.mkString(flakeRef.to_string(), state.mem); + v.mkString(flakeRef.to_string(), context, state.mem); } nix::PrimOp flakeRefToString({ @@ -181,7 +206,6 @@ nix::PrimOp flakeRefToString({ ``` )", .impl = prim_flakeRefToString, - .experimentalFeature = Xp::Flakes, }); } // namespace nix::flake::primops diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 67522d2c4751..0ba33ac326f0 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "nix/util/terminal.hh" #include "nix/util/ref.hh" @@ -36,9 +35,11 @@ #include "nix/expr/value-to-json.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" @@ -57,7 +58,6 @@ #include "nix/util/logging.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/position.hh" #include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" @@ -65,21 +65,22 @@ namespace nix { struct SourceAccessor; -using namespace flake; +using namespace fetchers; namespace flake { static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { - if (value.isThunk() && value.isTrivial()) + if (value.isTrivial()) state.forceValue(value, pos); } static void expectType(EvalState & state, ValueType type, Value & value, const PosIdx pos) { forceTrivialValue(state, value, pos); - if (value.type() != type) - throw Error("expected %s but got %s at %s", showType(type), showType(value.type()), state.positions[pos]); + auto t = value.type(); + if (t != type) + throw Error("expected %s but got %s at %s", showType(type), showType(t), state.positions[pos]); } static std::pair, fetchers::Attrs> parseFlakeInputs( @@ -90,7 +91,7 @@ static std::pair, fetchers::Attrs> parseFlakeInput const SourcePath & flakeDir, bool allowSelf); -static void parseFlakeInputAttr(EvalState & state, const Attr & attr, fetchers::Attrs & attrs) +static void parseFlakeInputAttr(EvalState & state, const nix::Attr & attr, fetchers::Attrs & attrs) { // Allow selecting a subset of enum values #pragma GCC diagnostic push @@ -144,6 +145,7 @@ static FlakeInput parseFlakeInput( auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sBuildTime = state.symbols.create("buildTime"); fetchers::Attrs attrs; std::optional url; @@ -172,6 +174,11 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); + } else if (attr.name == sBuildTime) { + expectType(state, nBool, *attr.value, attr.pos); + input.buildTime = attr.value->boolean(); + if (input.buildTime) + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; @@ -240,7 +247,7 @@ static std::pair, fetchers::Attrs> parseFlakeInput return {inputs, selfAttrs}; } -static Flake readFlake( +Flake readFlake( EvalState & state, const FlakeRef & originalRef, const FlakeRef & resolvedRef, @@ -260,6 +267,7 @@ static Flake readFlake( .resolvedRef = resolvedRef, .lockedRef = lockedRef, .path = flakePath, + .provenance = flakePath.getProvenance(), }; if (auto description = vInfo.attrs()->get(state.s.description)) { @@ -369,7 +377,8 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, - const InputAttrPath & lockRootAttrPath) + const InputAttrPath & lockRootAttrPath, + bool requireLockable) { // Fetch a lazy tree first. auto cachedInput = @@ -401,13 +410,14 @@ static Flake getFlake( originalRef, resolvedRef, lockedRef, - state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), + state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)), lockRootAttrPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) +Flake getFlake( + EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable) { - return getFlake(state, originalRef, useRegistries, {}); + return getFlake(state, originalRef, useRegistries, {}, requireLockable); } static LockFile readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) @@ -416,19 +426,13 @@ static LockFile readLockFile(const fetchers::Settings & fetchSettings, const Sou : LockFile(); } -/* Compute an in-memory lock file for the specified top-level flake, - and optionally write it to file, if the flake is writable. */ -LockedFlake -lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { - experimentalFeatureSettings.require(Xp::Flakes); - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistriesTop, {}); - if (lockFlags.applyNixConfig) { flake.config.apply(settings); state.store->setOptions(); @@ -607,7 +611,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); } else { - return getFlake(state, ref, useRegistries, inputAttrPath); + return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); } }; @@ -630,7 +634,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake, oldLock->parentInputAttrPath); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->buildTime, + oldLock->parentInputAttrPath); node->inputs.insert_or_assign(id, childNode); @@ -719,12 +727,34 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; + /* Warn against the use of indirect flakerefs + (but only at top-level since we don't want + to annoy users about flakes that are not + under their control). */ + auto warnRegistry = [&](const FlakeRef & resolvedRef) { + if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + inputAttrPathS, + flake.path, + s.str()); + } + }; + if (input.isFlake) { auto inputFlake = getInputFlake( *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); - auto childNode = - make_ref(inputFlake.lockedRef, ref, true, overriddenParentPath); + auto childNode = make_ref( + inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -746,6 +776,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, inputAttrPath, inputFlake.path, false); + + warnRegistry(inputFlake.resolvedRef); } else { @@ -757,16 +789,21 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto cachedInput = state.inputCache->getAccessor( state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); + auto resolvedRef = + FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); + warnRegistry(resolvedRef); + return { - state.storePath( - state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor)), + state.storePath(state.mountInput( + lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), lockedRef}; } }(); - auto childNode = make_ref(lockedRef, ref, false, overriddenParentPath); + auto childNode = + make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); nodePaths.emplace(childNode, path); @@ -876,15 +913,13 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; - flake = getFlake(state, topRef, useRegistriesTop); + flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) @@ -908,6 +943,23 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, } } +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +{ + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; + + return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {}, false)); +} + +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags) +{ + /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */ + auto fakeRef = parseFlakeRef(state.fetchSettings, "flake:get-flake"); + return lockFlake(settings, state, fakeRef, lockFlags, readFlake(state, fakeRef, fakeRef, fakeRef, flakeDir, {})); +} + static ref makeInternalFS() { auto internalFS = make_ref(MemorySourceAccessor{}); @@ -931,8 +983,6 @@ static Value * requireInternalFile(EvalState & state, CanonPath path) void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { - experimentalFeatureSettings.require(Xp::Flakes); - auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); @@ -969,10 +1019,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) auto vLocks = state.allocValue(); vLocks->mkString(lockFileStr, state.mem); - auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree"); - assert(vFetchFinalTree); - - Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree}; + Value * args[] = {vLocks, &vOverrides}; state.callFunction(*vCallFlake, args, vRes, noPos); } @@ -987,11 +1034,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); @@ -1003,41 +1063,6 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc Flake::~Flake() {} -ref openEvalCache(EvalState & state, ref lockedFlake) -{ - auto fingerprint = state.settings.useEvalCache && state.settings.pureEval - ? lockedFlake->getFingerprint(*state.store, state.fetchSettings) - : std::nullopt; - auto rootLoader = [&state, lockedFlake]() { - /* For testing whether the evaluation cache is - complete. */ - if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") - throw Error("not everything is cached, but evaluation is not allowed"); - - auto vFlake = state.allocValue(); - callFlake(state, *lockedFlake, *vFlake); - - state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - return aOutputs->value; - }; - - if (fingerprint) { - auto search = state.evalCaches.find(fingerprint.value()); - if (search == state.evalCaches.end()) { - search = state.evalCaches - .emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)) - .first; - } - return search->second; - } else { - return make_ref(std::nullopt, state, rootLoader); - } -} - } // namespace flake } // namespace nix diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index c35040adb5d8..3af499ae6b80 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -26,6 +26,7 @@ #include "nix/store/outputs-spec.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix { class Store; @@ -42,12 +43,12 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; #endif -std::string FlakeRef::to_string() const +std::string FlakeRef::to_string(bool abbreviate) const { StringMap extraQuery; if (subdir != "") extraQuery.insert_or_assign("dir", subdir); - return input.toURLString(extraQuery); + return input.toURLString(extraQuery, abbreviate); } fetchers::Attrs FlakeRef::toAttrs() const @@ -90,7 +91,8 @@ static std::pair fromParsedURL(const fetchers::Settings & fetchSettings, ParsedURL && parsedURL, bool isFlake) { auto dir = getOr(parsedURL.query, "dir", ""); - parsedURL.query.erase("dir"); + if (!fetchSettings.nix219Compat) + parsedURL.query.erase("dir"); std::string fragment; std::swap(fragment, parsedURL.fragment); @@ -146,7 +148,7 @@ std::pair parsePathFlakeRefWithFragment( // Save device to detect filesystem boundary dev_t device = lstat(path).st_dev; bool found = false; - while (path != "/") { + while (path.parent_path() != path) { if (pathExists(path / "flake.nix")) { found = true; break; @@ -171,7 +173,7 @@ std::pair parsePathFlakeRefWithFragment( auto flakeRoot = path; std::string subdir; - while (flakeRoot != "/") { + while (flakeRoot.parent_path() != flakeRoot) { if (pathExists(flakeRoot / ".git")) { auto parsedURL = ParsedURL{ .scheme = "git+file", @@ -270,7 +272,7 @@ std::pair parseFlakeRefWithFragment( bool isFlake, bool preserveRelativePaths) { - using namespace fetchers; + using namespace nix::fetchers; if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 3bee6556f643..99720c9403a0 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -10,6 +10,7 @@ namespace nix { class EvalState; +struct Provenance; namespace flake { @@ -44,12 +45,18 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; + /** - * true = process flake to get outputs - * - * false = (fetched) static source path + * Whether to call the `flake.nix` file in this input to get its outputs. */ bool isFlake = true; + + /** + * Whether to fetch this input at evaluation time or at build + * time. + */ + bool buildTime = false; + std::optional follows; FlakeInputs overrides; }; @@ -88,6 +95,11 @@ struct Flake */ SourcePath path; + /** + * Cached provenance of `flake.nix` (equivalent to `path.getProvenance()`). + */ + std::shared_ptr provenance; + /** * Pretend that `lockedRef` is dirty. */ @@ -116,7 +128,8 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries); +Flake getFlake( + EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries, bool requireLockable = true); /** * Fingerprint of a locked flake; used as a cache key. @@ -212,33 +225,40 @@ struct LockFlags * for those inputs will be ignored. */ std::set inputUpdates; + + /** + * Whether to require a locked input. + */ + bool requireLockable = true; }; +/** + * Return a `Flake` object representing the flake read from the + * `flake.nix` file in `rootDir`. + */ +Flake readFlake( + EvalState & state, + const FlakeRef & originalRef, + const FlakeRef & resolvedRef, + const FlakeRef & lockedRef, + const SourcePath & rootDir, + const InputAttrPath & lockRootPath); + +/* + * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is + * writable. + */ LockedFlake lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); -void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); +LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake); -/** - * Open an evaluation cache for a flake. - */ -ref openEvalCache(EvalState & state, ref lockedFlake); - -} // namespace flake +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); -void emitTreeAttrs( - EvalState & state, - const StorePath & storePath, - const fetchers::Input & input, - Value & v, - bool emptyRevFallback = false, - bool forceDirty = false); +void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); -/** - * An internal builtin similar to `fetchTree`, except that it - * always treats the input as final (i.e. no attributes can be - * added/removed/changed). - */ -void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Value & v); +} // namespace flake } // namespace nix diff --git a/src/libflake/include/nix/flake/flakeref.hh b/src/libflake/include/nix/flake/flakeref.hh index b557433a9fb8..1f39d62ebbf9 100644 --- a/src/libflake/include/nix/flake/flakeref.hh +++ b/src/libflake/include/nix/flake/flakeref.hh @@ -68,8 +68,7 @@ struct FlakeRef { } - // FIXME: change to operator <<. - std::string to_string() const; + std::string to_string(bool abbreviate = false) const; fetchers::Attrs toAttrs() const; diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index 89029aec491c..27232b20a669 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -111,6 +111,7 @@ struct LockedNode : Node { FlakeRef lockedRef, originalRef; bool isFlake = true; + bool buildTime = false; /* The node relative to which relative source paths (e.g. 'path:../foo') are interpreted. */ @@ -120,10 +121,12 @@ struct LockedNode : Node const FlakeRef & lockedRef, const FlakeRef & originalRef, bool isFlake = true, + bool buildTime = false, std::optional parentInputAttrPath = {}) : lockedRef(std::move(lockedRef)) , originalRef(std::move(originalRef)) , isFlake(isFlake) + , buildTime(buildTime) , parentInputAttrPath(std::move(parentInputAttrPath)) { } diff --git a/src/libflake/include/nix/flake/meson.build b/src/libflake/include/nix/flake/meson.build index fc580164eaec..fbe54f41208b 100644 --- a/src/libflake/include/nix/flake/meson.build +++ b/src/libflake/include/nix/flake/meson.build @@ -6,6 +6,7 @@ headers = files( 'flake.hh', 'flakeref.hh', 'lockfile.hh', + 'provenance.hh', 'settings.hh', 'url-name.hh', ) diff --git a/src/libflake/include/nix/flake/provenance.hh b/src/libflake/include/nix/flake/provenance.hh new file mode 100644 index 000000000000..011744f5e65d --- /dev/null +++ b/src/libflake/include/nix/flake/provenance.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "nix/util/provenance.hh" + +namespace nix { + +struct FlakeProvenance : Provenance +{ + std::shared_ptr next; + std::string flakeOutput; + bool pure = true; + + FlakeProvenance(std::shared_ptr next, std::string flakeOutput, bool pure) + : next(std::move(next)) + , flakeOutput(std::move(flakeOutput)) + , pure(pure) {}; + + nlohmann::json to_json() const override; +}; + +} // namespace nix diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 7187a3294a34..05b36f5b779c 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -21,13 +21,7 @@ struct Settings : public Config void configureEvalSettings(nix::EvalSettings & evalSettings) const; Setting useRegistries{ - this, - true, - "use-registries", - "Whether to use flake registries to resolve flake references.", - {}, - true, - Xp::Flakes}; + this, true, "use-registries", "Whether to use flake registries to resolve flake references.", {}, true}; Setting acceptFlakeConfig{ this, @@ -35,8 +29,7 @@ struct Settings : public Config "accept-flake-config", "Whether to accept Nix configuration settings from a flake without prompting.", {}, - true, - Xp::Flakes}; + true}; Setting commitLockFileSummary{ this, @@ -47,8 +40,7 @@ struct Settings : public Config empty, the summary is generated based on the action performed. )", {"commit-lockfile-summary"}, - true, - Xp::Flakes}; + true}; }; } // namespace nix::flake diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 4eeba3bd3ef1..c44ae77dff91 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -8,10 +8,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -32,10 +29,9 @@ #include "nix/flake/flakeref.hh" #include "nix/store/path.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/fmt.hh" -#include "nix/util/hash.hh" +#include "nix/util/json-utils.hh" #include "nix/util/logging.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" @@ -71,6 +67,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann: : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) , parentInputAttrPath( json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { @@ -229,13 +226,11 @@ std::pair LockFile::toJSON() const if (auto lockedNode = node.dynamic_pointer_cast()) { n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - /* For backward compatibility, omit the "__final" - attribute. We never allow non-final inputs in lock files - anyway. */ assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative()); - n["locked"].erase("__final"); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; if (lockedNode->parentInputAttrPath) n["parent"] = *lockedNode->parentInputAttrPath; } @@ -352,7 +347,7 @@ std::map LockFile::getAllInputs() const static std::string describe(const FlakeRef & flakeRef) { - auto s = fmt("'%s'", flakeRef.to_string()); + auto s = fmt("'%s'", flakeRef.to_string(true)); if (auto lastModified = flakeRef.input.getLastModified()) s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 58916ecd9ab2..1bf6f7e5fba6 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -45,6 +45,7 @@ sources = files( 'flake.cc', 'flakeref.cc', 'lockfile.cc', + 'provenance.cc', 'settings.cc', 'url-name.cc', ) @@ -58,11 +59,10 @@ this_library = library( 'nixflake', sources, generated_headers, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, ) diff --git a/src/libflake/package.nix b/src/libflake/package.nix index dd442a44ec9a..2b0c827a09ce 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake"; + pname = "determinate-nix-flake"; inherit version; workDir = ./.; diff --git a/src/libflake/provenance.cc b/src/libflake/provenance.cc new file mode 100644 index 000000000000..c80c4154561a --- /dev/null +++ b/src/libflake/provenance.cc @@ -0,0 +1,28 @@ +#include "nix/flake/provenance.hh" +#include "nix/util/json-utils.hh" + +#include + +namespace nix { + +nlohmann::json FlakeProvenance::to_json() const +{ + return nlohmann::json{ + {"type", "flake"}, + {"next", next ? next->to_json() : nlohmann::json(nullptr)}, + {"flakeOutput", flakeOutput}, + {"pure", pure}}; +} + +Provenance::Register registerFlakeProvenance("flake", [](nlohmann::json json) { + auto & obj = getObject(json); + std::shared_ptr next; + if (auto p = optionalValueAt(obj, "next"); p && !p->is_null()) + next = Provenance::from_json(*p); + bool pure = true; + if (auto p = optionalValueAt(obj, "pure")) + pure = getBoolean(*p); + return make_ref(next, getString(valueAt(obj, "flakeOutput")), pure); +}); + +} // namespace nix diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 36332fdb70a1..5b9106f455b9 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -42,16 +42,32 @@ headers = files( subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixmainc', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixmainc', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixmainc', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libmain-c/meson.options b/src/libmain-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libmain-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc index 0ee965dc82ee..872949e423b9 100644 --- a/src/libmain-c/nix_api_main.cc +++ b/src/libmain-c/nix_api_main.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index f019a917d360..dd2faa913300 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -10,6 +10,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -17,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main-c"; + pname = "determinate-nix-main-c"; inherit version; workDir = ./.; @@ -27,7 +28,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -41,6 +42,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c47..b20df3a99ec9 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index f9e771205ba2..0416fc080343 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -26,7 +26,9 @@ void parseCmdLine( const Strings & args, fun parseArg); -void printVersion(const std::string & programName); +std::string version(); + +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index a3e75c535dd0..a1c3768f2b15 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -50,7 +50,8 @@ void setLogFormat(const std::string & logFormatStr) void setLogFormat(const LogFormat & logFormat) { defaultLogFormat = logFormat; - logger = makeDefaultLogger(); + logger->stop(); + logger = makeDefaultLogger().release(); } } // namespace nix diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 2ac59924e592..97b89b9fa79a 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -77,11 +77,10 @@ this_library = library( 'nixmain', sources, config_priv_h, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, ) diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7b0a4dee7dad..119e1f1aca59 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main"; + pname = "determinate-nix-main"; inherit version; workDir = ./.; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a973102f9509..48d44a3a4b81 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,10 +1,12 @@ #include "nix/main/progress-bar.hh" #include "nix/util/terminal.hh" #include "nix/util/sync.hh" -#include "nix/store/store-api.hh" +#include "nix/util/signals.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" -#include #include #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -51,6 +62,7 @@ class ProgressBar : public Logger ActivityId parent; std::optional name; std::chrono::time_point startTime; + bool logged = false; }; struct ActivitiesByType @@ -94,12 +106,55 @@ class ProgressBar : public Logger bool printBuildLogs = false; bool isTTY; + std::unique_ptr interruptCallback, stopCallback, contCallback, winchCallback; + + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) : isTTY(isTTY) + , interruptCallback(createInterruptCallback([&]() { + pause(); + redraw("\rshutting down\e[K"); + })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; + + /* On Ctrl-Z, unhide the cursor before the process is + stopped. */ + stopCallback = createSignalCallback(SignalType::Stop, [&]() { + auto state(state_.lock()); + if (state->active && !state->isPaused()) + unhideCursorIfNeeded(); + }); + + /* Redraw the progress bar when the process is resumed or the + terminal size changes. */ + auto redrawCallback = [&]() { + auto state(state_.lock()); + if (state->active && !state->isPaused()) { + /* Force a redraw, since the new output may be + identical to the one cached by redraw(). */ + invalidateRedrawCache(); + hideCursorIfNeeded(); + update(*state); + } + }; + contCallback = createSignalCallback(SignalType::Cont, redrawCallback); + winchCallback = createSignalCallback(SignalType::Winch, redrawCallback); + updateThread = std::thread([&]() { auto state(state_.lock()); auto nextWakeup = std::chrono::milliseconds::max(); @@ -124,7 +179,8 @@ class ProgressBar : public Logger auto state(state_.lock()); if (state->active) { state->active = false; - writeToStderr("\r\e[K"); + clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -142,8 +198,15 @@ class ProgressBar : public Logger return; } - if (state->active) - writeToStderr("\r\e[K"); + if (state->active) { + clearProgressDisplay(); + /* Show activities that were previously only shown on the + progress bar. Otherwise the user won't know what's + happening. */ + for (auto & act : state->activities) + logActivity(*state, lvlNotice, act); + unhideCursorIfNeeded(); + } } void resume() override @@ -156,8 +219,10 @@ class ProgressBar : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) - writeToStderr("\r\e[K"); + if (state->active) { + clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -168,7 +233,7 @@ class ProgressBar : public Logger return printBuildLogs; } - void log(Verbosity lvl, std::string_view s) override + void log(Verbosity lvl, std::string_view s) noexcept override { if (lvl > verbosity) return; @@ -176,7 +241,7 @@ class ProgressBar : public Logger log(*state, lvl, s); } - void logEI(const ErrorInfo & ei) override + void logEI(const ErrorInfo & ei) noexcept override { auto state(state_.lock()); @@ -186,9 +251,10 @@ class ProgressBar : public Logger log(*state, ei.level, oss.view()); } - void log(State & state, Verbosity lvl, std::string_view s) + void log(State & state, Verbosity lvl, std::string_view s) noexcept { if (state.active) { + invalidateRedrawCache(); writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { @@ -196,39 +262,38 @@ class ProgressBar : public Logger } } + void logActivity(State & state, Verbosity lvl, ActInfo & act) noexcept + { + if (!act.logged && lvl <= verbosity && !act.s.empty() && act.type != actBuildWaiting) { + log(state, lvl, act.s + "..."); + act.logged = true; + } + } + void startActivity( ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, - ActivityId parent) override + ActivityId parent) noexcept override { auto state(state_.lock()); - if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) - log(*state, lvl, s + "..."); - state->activities.emplace_back( ActInfo{.s = s, .type = type, .parent = parent, .startTime = std::chrono::steady_clock::now()}); auto i = std::prev(state->activities.end()); state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); + logActivity(*state, lvl, *i); + if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); - - // Used to be curRound and nrRounds, but the - // implementation was broken for a long time. - if (getI(fields, 2) != 1 || getI(fields, 3) != 1) { - throw Error("log message indicated repeating builds, but this is not currently implemented"); - } i->name = DrvName(name).name; } @@ -243,9 +308,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -278,7 +341,7 @@ class ProgressBar : public Logger return false; } - void stopActivity(ActivityId act) override + void stopActivity(ActivityId act) noexcept override { auto state(state_.lock()); @@ -300,7 +363,7 @@ class ProgressBar : public Logger update(*state); } - void result(ActivityId act, ResultType type, const std::vector & fields) override + void result(ActivityId act, ResultType type, const std::vector & fields) noexcept override { auto state(state_.lock()); @@ -379,7 +442,7 @@ class ProgressBar : public Logger } } - void update(State & state) + void update(State & state) noexcept { state.haveUpdate = true; updateCV.notify_one(); @@ -392,7 +455,7 @@ class ProgressBar : public Logger * with text selection in some terminals, including libvte-based terminal * emulators. */ - void redraw(std::string newOutput) + void redraw(std::string newOutput) noexcept { auto lastOutput(lastOutput_.lock()); if (newOutput != *lastOutput) { @@ -401,7 +464,18 @@ class ProgressBar : public Logger } } - std::chrono::milliseconds draw(State & state) + void invalidateRedrawCache() + { + *lastOutput_.lock() = ""; + } + + void clearProgressDisplay() + { + invalidateRedrawCache(); + writeToStderr("\r\e[K"); + } + + std::chrono::milliseconds draw(State & state) noexcept { auto nextWakeup = std::chrono::milliseconds::max(); @@ -456,16 +530,12 @@ class ProgressBar : public Logger } } - auto width = getWindowSize().second; - if (width <= 0) - width = std::numeric_limits::max(); - - redraw("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + redraw("\r" + filterANSIEscapes(line, false, getWindowWidth()) + ANSI_NORMAL + "\e[K"); return nextWakeup; } - std::string getStatus(State & state) + std::string getStatus(State & state) noexcept { std::string res; @@ -640,6 +710,7 @@ class ProgressBar : public Logger { auto state(state_.lock()); if (state->active) { + invalidateRedrawCache(); std::cerr << "\r\e[K"; Logger::writeToStdout(s); draw(*state); @@ -653,8 +724,11 @@ class ProgressBar : public Logger auto state(state_.lock()); if (!state->active) return {}; + invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); @@ -667,6 +741,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8a74484d451a..abf6e49e11b7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,12 +6,10 @@ #include "nix/store/store-open.hh" #include "nix/store/gc-store.hh" #include "nix/main/loggers.hh" -#include "nix/main/progress-bar.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" #include -#include #include #include @@ -23,6 +21,9 @@ #ifndef _WIN32 # include #endif +#ifdef __APPLE__ +# include +#endif #ifdef __linux__ # include #endif @@ -65,8 +66,7 @@ void printMissing(ref store, const MissingPaths & missing, Verbosity lvl) else printMsg(lvl, "these %d derivations will be built:", missing.willBuild.size()); auto sorted = store->topoSortPaths(missing.willBuild); - reverse(sorted.begin(), sorted.end()); - for (auto & i : sorted) + for (auto & i : sorted | std::views::reverse) printMsg(lvl, " %s", store->printStorePath(i)); } @@ -134,15 +134,26 @@ void bumpFileLimit() if (getrlimit(RLIMIT_NOFILE, &limit) != 0) return; - if (limit.rlim_cur < limit.rlim_max) { - // Some software misbehaves really bad when we try to raise the - // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used - // by the daemon. - // - // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. - // This was fixed in: - // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 - limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); + rlim_t target = limit.rlim_max; + +# ifdef __APPLE__ + // On macOS the hard limit is typically RLIM_INFINITY, but + // setting rlim_cur to that causes problems: child processes + // (e.g. GNU patch in the Nix sandbox) may allocate memory + // proportional to the fd limit and OOM. Use the kernel's + // per-process file limit instead, which is the effective cap. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + int maxfiles; + size_t len = sizeof(maxfiles); + if (sysctlbyname("kern.maxfilesperproc", &maxfiles, &len, nullptr, 0) == 0) + target = maxfiles; +# endif + + if (limit.rlim_cur < target) { + limit.rlim_cur = target; // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } @@ -171,10 +182,19 @@ void initNix(bool loadConfig) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); - /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + /* Install a dummy NIX_SIG_MULTI_INT handler for use with pthread_kill(). */ act.sa_handler = sigHandler; - if (sigaction(SIGUSR1, &act, 0)) - throw SysError("handling SIGUSR1"); + if (sigaction(NIX_SIG_MULTI_INT, &act, 0)) + throw SysError("handling multiplexed interrupt"); + + /* Reset SIGQUIT to its default disposition. In particular, this + unregisters any crash handler installed by `sentry_init()` + (which runs before us): SIGQUIT is a user-initiated "quit with + core dump" action (e.g. Ctrl-\ at a terminal), not a crash, so + it should not be reported. */ + act.sa_handler = SIG_DFL; + if (sigaction(SIGQUIT, &act, 0)) + throw SysError("handling SIGQUIT"); #endif #ifdef __APPLE__ @@ -185,6 +205,13 @@ void initNix(bool loadConfig) if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + /* Same for SIGCONT and SIGTSTP, which are also handled by + * signalHandlerThread. */ + if (sigaction(SIGCONT, &act, 0)) + throw SysError("handling SIGCONT"); + if (sigaction(SIGTSTP, &act, 0)) + throw SysError("handling SIGTSTP"); + /* Disable SA_RESTART for interrupts, so that system calls on this thread * error with EINTR like they do on Linux. * Most signals on BSD systems default to SA_RESTART on, but Nix @@ -198,8 +225,6 @@ void initNix(bool loadConfig) throw SysError("handling SIGHUP"); if (sigaction(SIGPIPE, &act, 0)) throw SysError("handling SIGPIPE"); - if (sigaction(SIGQUIT, &act, 0)) - throw SysError("handling SIGQUIT"); if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif @@ -325,9 +350,14 @@ void parseCmdLine( LegacyArgs(programName, parseArg).parseCmdline(args); } +std::string version() +{ + return fmt("(Determinate Nix %s) %s", determinateNixVersion, nixVersion); +} + void printVersion(const std::string & programName) { - std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; + std::cout << fmt("%s %s", programName, version()) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if NIX_USE_BOEHMGC @@ -337,12 +367,12 @@ void printVersion(const std::string & programName) std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "System configuration file: " << nixConfFile() << "\n"; + std::cout << "System configuration file: " << os_string_to_string(nixConfFile().native()) << "\n"; std::cout << "User configuration files: " << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n"; std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir << "\n"; - std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "State directory: " << os_string_to_string(settings.nixStateDir.native()) << "\n"; } throw Exit(); } diff --git a/src/libmain/unix/stack.cc b/src/libmain/unix/stack.cc index bec0d389f5ed..e049bae50de1 100644 --- a/src/libmain/unix/stack.cc +++ b/src/libmain/unix/stack.cc @@ -10,6 +10,8 @@ namespace nix { +static struct sigaction savedSigsegvAction; + static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) { /* Detect stack overflows by comparing the faulting address with @@ -34,12 +36,8 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) } } - /* Restore default behaviour (i.e. segfault and dump core). */ - struct sigaction act; - sigfillset(&act.sa_mask); - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - if (sigaction(SIGSEGV, &act, 0)) + /* Restore the original SIGSEGV handler. */ + if (sigaction(SIGSEGV, &savedSigsegvAction, 0)) abort(); } @@ -63,7 +61,7 @@ void detectStackOverflow() sigfillset(&act.sa_mask); act.sa_sigaction = sigsegvHandler; act.sa_flags = SA_SIGINFO | SA_ONSTACK; - if (sigaction(SIGSEGV, &act, 0)) + if (sigaction(SIGSEGV, &act, &savedSigsegvAction)) throw SysError("resetting SIGSEGV"); #endif } diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index c81235bf16d4..4099e6931316 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -45,16 +45,32 @@ headers += files('nix_api_store_internal.h') subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') -this_library = library( - 'nixstorec', - sources, - soversion : nix_soversion, - dependencies : deps_public + deps_private + deps_other, - include_directories : include_dirs, - link_args : linker_export_flags, - prelink : true, # For C++ static initializers - install : true, -) +# For linking -c bindings into the cli for plugins. +build_both_libraries = get_option('plugin-c-api') + +library_kwargs = { + 'dependencies' : deps_public + deps_private + deps_other, + 'include_directories' : include_dirs, + 'link_args' : linker_export_flags, + 'install' : true, +} + +if build_both_libraries + this_libraries = both_libraries( + 'nixstorec', + sources, + kwargs : library_kwargs, + override_options : [ 'b_lto=false' ], + ) +else + this_library = library( + 'nixstorec', + sources, + kwargs : library_kwargs, + ) +endif + +plugin_c_api_enabled = build_both_libraries install_headers(headers, preserve_path : true) diff --git a/src/libstore-c/meson.options b/src/libstore-c/meson.options new file mode 100644 index 000000000000..a8b0c4df0401 --- /dev/null +++ b/src/libstore-c/meson.options @@ -0,0 +1,8 @@ +# vim: filetype=meson + +option( + 'plugin-c-api', + type : 'boolean', + value : false, + yield : true, +) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fbb3c418566c..f0d8aeab3aa1 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -289,6 +289,29 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store NIXC_CATCH_ERRS_NULL } +nix_err nix_derivation_make_outputs( + nix_c_context * context, + Store * store, + const char * json, + void (*callback)(void * userdata, const char * output_name, const char * path), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto drv = nix::Derivation::parseJsonAndValidate(*store->ptr, nlohmann::json::parse(json)); + + for (auto & output : drv.outputs) { + auto outPath = output.second.path(*store->ptr, drv.name, output.first); + + if (callback && outPath) { + callback(userdata, output.first.c_str(), store->ptr->printStorePath(*outPath).c_str()); + } + } + } + NIXC_CATCH_ERRS +} + nix_err nix_derivation_to_json( nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata) { @@ -342,6 +365,96 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * NIXC_CATCH_ERRS_NULL } +nix_err nix_store_drv_from_path( + nix_c_context * context, + Store * store, + const StorePath * path, + void (*callback)(void * userdata, const nix_derivation * drv), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Derivation drv = store->ptr->derivationFromPath(path->path); + if (callback) { + const nix_derivation tmp{drv, store}; + callback(userdata, &tmp); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_store_query_path_info( + nix_c_context * context, + Store * store, + const StorePath * store_path, + void * userdata, + nix_get_string_callback callback) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto info = store->ptr->queryPathInfo(store_path->path); + if (callback) { + auto result = info->toJSON(&store->ptr->config, true, nix::PathInfoJsonFormat::V1).dump(); + callback(result.data(), result.size(), userdata); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_store_build_paths( + nix_c_context * context, + Store * store, + const StorePath ** store_paths, + unsigned int num_store_paths, + void (*callback)(void * userdata, const char * path, const char * result), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::vector derived_paths; + for (size_t i = 0; i < num_store_paths; i++) { + const StorePath * store_path = store_paths[i]; + derived_paths.push_back(nix::SingleDerivedPath::Opaque{store_path->path}); + } + + auto results = store->ptr->buildPathsWithResults(derived_paths); + for (auto & result : results) { + if (callback) + callback( + userdata, result.path.to_string(store->ptr->config).c_str(), nlohmann::json(result).dump().c_str()); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_derivation_get_outputs_and_optpaths( + nix_c_context * context, + const nix_derivation * drv, + const Store * store, + void (*callback)(void * userdata, const char * name, const StorePath * path), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto value = drv->drv.outputsAndOptPaths(store->ptr->config); + if (callback) { + for (const auto & [name, result] : value) { + if (auto store_path = result.second) { + const StorePath tmp_path{*store_path}; + callback(userdata, name.c_str(), &tmp_path); + } else { + callback(userdata, name.c_str(), nullptr); + } + } + } + } + NIXC_CATCH_ERRS +} + StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index e60a7d6c0f7c..964c4066154b 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -201,6 +201,22 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal */ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json); +/** + * @brief Hashes the derivation and gives the output paths + * + * @param[in] context Optional, stores error information. + * @param[in] store nix store reference. + * @param[in] json JSON of the derivation as a string. + * @param[in] callback Called for every output to provide the output path. + * @param[in] userdata User data to pass to the callback. + */ +nix_err nix_derivation_make_outputs( + nix_c_context * context, + Store * store, + const char * json, + void (*callback)(void * userdata, const char * output_name, const char * path), + void * userdata); + /** * @brief Add the given `nix_derivation` to the given store * @@ -259,6 +275,88 @@ nix_err nix_store_get_fs_closure( */ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path); +/** + * @note The callback borrows the Derivation only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] store The nix store + * @param[in] path The nix store path + * @param[in] callback The callback to call + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_store_drv_from_path( + nix_c_context * context, + Store * store, + const StorePath * path, + void (*callback)(void * userdata, const nix_derivation * drv), + void * userdata); + +/** + * @brief Queries for the nix store path info. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_path A store path + * @param[in] userdata The data to pass to the callback + * @param[in] callback Called for when the path info is resolved + */ +nix_err nix_store_query_path_info( + nix_c_context * context, + Store * store, + const StorePath * store_path, + void * userdata, + nix_get_string_callback callback); + +/** + * @brief Builds the paths, if they are a derivation then they get built. + * + * @note Path and result for the callback only exist for the lifetime of + * the call. Result is a string containing the build result in JSON. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_paths Pointer to list of nix store paths + * @param[in] num_store_paths Number of nix store paths + * @param[in] callback The callback to trigger for build results + * @param[in] userdata User data to pass to the callback + */ +nix_err nix_store_build_paths( + nix_c_context * context, + Store * store, + const StorePath ** store_paths, + unsigned int num_store_paths, + void (*callback)(void * userdata, const char * path, const char * result), + void * userdata); + +/** + * @brief Iterate and get all of the store paths for each output. + * + * @note The callback borrows the StorePath only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] drv The derivation + * @param[in] store The nix store + * @param[in] callback The function to call on every output and store path + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_derivation_get_outputs_and_optpaths( + nix_c_context * context, + const nix_derivation * drv, + const Store * store, + void (*callback)(void * userdata, const char * name, const StorePath * path), + void * userdata); + +/** + * @brief Gets the derivation as a JSON string + * + * @param[out] context Optional, stores error information + * @param[in] drv The derivation + * @param[in] callback Called with the JSON string + * @param[in] userdata Arbitrary data passed to the callback + */ +nix_err nix_derivation_to_json( + nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata); + /** * @brief Query the full store path given the hash part of a valid store * path, or empty if no matching path is found. diff --git a/src/libstore-c/nix_api_store_internal.h b/src/libstore-c/nix_api_store_internal.h index 712d96488a57..0199628da8a9 100644 --- a/src/libstore-c/nix_api_store_internal.h +++ b/src/libstore-c/nix_api_store_internal.h @@ -18,6 +18,7 @@ struct StorePath struct nix_derivation { nix::Derivation drv; + Store * store; }; } // extern "C" diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fde17c78e017..ecfc278cf660 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -8,6 +8,7 @@ # Configuration Options version, + withPluginCAPI, }: let @@ -15,7 +16,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-c"; + pname = "determinate-nix-store-c"; inherit version; workDir = ./.; @@ -25,7 +26,7 @@ mkMesonLibrary (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) (fileset.fileFilter (file: file.hasExt "h") ./.) @@ -37,6 +38,7 @@ mkMesonLibrary (finalAttrs: { ]; mesonFlags = [ + (lib.mesonBool "plugin-c-api" withPluginCAPI) ]; meta = { diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index cb1d23ac5208..c27edc95ef9b 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,6 +1,4 @@ -#include - -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc index 79548f61f3fe..549e3ce36172 100644 --- a/src/libstore-test-support/https-store.cc +++ b/src/libstore-test-support/https-store.cc @@ -60,7 +60,7 @@ void HttpsBinaryCacheStoreTest::SetUp() openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"}); // clang-format on -#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */ +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ auto args = serverArgs(); serverPid = startProcess( [&] { @@ -91,7 +91,9 @@ void HttpsBinaryCacheStoreTest::SetUp() void HttpsBinaryCacheStoreTest::TearDown() { +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ serverPid.kill(); +#endif delTmpDir.reset(); testFileTransferSettings.reset(); } diff --git a/src/libstore-test-support/include/nix/store/tests/derived-path.hh b/src/libstore-test-support/include/nix/store/tests/derived-path.hh index b3b43474a914..7d93af08fd1f 100644 --- a/src/libstore-test-support/include/nix/store/tests/derived-path.hh +++ b/src/libstore-test-support/include/nix/store/tests/derived-path.hh @@ -9,36 +9,35 @@ #include "nix/store/tests/outputs-spec.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh index 3aa2cd3455e9..9d4db058e5a4 100644 --- a/src/libstore-test-support/include/nix/store/tests/https-store.hh +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -43,7 +43,7 @@ class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig { public: TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , HttpBinaryCacheStoreConfig(url, params) { } @@ -65,7 +65,9 @@ protected: std::filesystem::path tmpDir, cacheDir; std::filesystem::path caCert, caKey, serverCert, serverKey; std::filesystem::path clientCert, clientKey; +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ Pid serverPid; +#endif uint16_t port = 8443; std::shared_ptr localCacheStore; diff --git a/src/libstore-test-support/include/nix/store/tests/libstore.hh b/src/libstore-test-support/include/nix/store/tests/libstore.hh index d79b5531232a..740cc2dcf427 100644 --- a/src/libstore-test-support/include/nix/store/tests/libstore.hh +++ b/src/libstore-test-support/include/nix/store/tests/libstore.hh @@ -10,6 +10,29 @@ namespace nix { +/** + * Scoped guard that enables an experimental feature for the lifetime + * of the object. + */ +struct EnableExperimentalFeature +{ + std::set previous; + + explicit EnableExperimentalFeature(std::string_view feature) + : previous(experimentalFeatureSettings.experimentalFeatures.get()) + { + experimentalFeatureSettings.set("extra-experimental-features", std::string{feature}); + } + + ~EnableExperimentalFeature() + { + experimentalFeatureSettings.experimentalFeatures.assign(previous); + } + + EnableExperimentalFeature(const EnableExperimentalFeature &) = delete; + EnableExperimentalFeature & operator=(const EnableExperimentalFeature &) = delete; +}; + class LibStoreTest : public virtual ::testing::Test { public: diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bb9e5a3038fd..df48a7469ea8 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; @@ -79,23 +76,23 @@ class nix_api_store_test : public nix_api_store_test_base { public: nix_api_store_test() - : nix_api_store_test_base{} - { - init_local_store(); - }; + : nix_api_store_test_base{} {}; - ~nix_api_store_test() override + void SetUp() override { - nix_store_free(store); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif + store = open_local_store(); } - Store * store; - -protected: - void init_local_store() + ~nix_api_store_test() override { - store = open_local_store(); + if (store) + nix_store_free(store); } + + Store * store = nullptr; }; } // namespace nixC diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index 5bbcc734086a..6cdb0a60ef96 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" @@ -9,12 +9,11 @@ #include "nix/store/tests/path.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/path.hh b/src/libstore-test-support/include/nix/store/tests/path.hh index ff80b1299a04..e2ecd23ca5f0 100644 --- a/src/libstore-test-support/include/nix/store/tests/path.hh +++ b/src/libstore-test-support/include/nix/store/tests/path.hh @@ -18,18 +18,17 @@ void showValue(const StorePath & p, std::ostream & os); } // namespace nix namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 563c8cfb6c4e..543facead0ec 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -122,4 +122,10 @@ public: VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \ VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink +{ + void writeUnbuffered(std::string_view data) override {} +}; + } // namespace nix diff --git a/src/libstore-test-support/libstore-network.cc b/src/libstore-test-support/libstore-network.cc index 8aa047bdd609..800dc17ed9f5 100644 --- a/src/libstore-test-support/libstore-network.cc +++ b/src/libstore-test-support/libstore-network.cc @@ -5,6 +5,8 @@ #ifdef __linux__ # include "nix/util/file-system.hh" # include "nix/util/linux-namespaces.hh" +# include +# include # include # include # include @@ -34,8 +36,9 @@ static void enterNetworkNamespace() if (!fd) throw SysError("cannot open IP socket for loopback interface"); + using namespace std::string_view_literals; struct ::ifreq ifr = {}; - strcpy(ifr.ifr_name, "lo"); + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 4d904cb1d06a..79f3c8235d2e 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -50,13 +50,12 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', sources, - soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 # is available. See also ../libutil/build.meson link_args : linker_export_flags + [ '-lrapidcheck' ], - prelink : true, # For C++ static initializers + prelink : prelink, # For C++ static initializers install : true, ) diff --git a/src/libstore-test-support/outputs-spec.cc b/src/libstore-test-support/outputs-spec.cc index d5128a8bd91a..947f02eb539f 100644 --- a/src/libstore-test-support/outputs-spec.cc +++ b/src/libstore-test-support/outputs-spec.cc @@ -3,10 +3,13 @@ #include namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using nix::OutputsSpec; + using nix::StorePathName; + using nix::StringSet; + return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { switch (n) { diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 4f20f2cbe716..bd5a4c0aa7a0 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -19,7 +19,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-test-support"; + pname = "determinate-nix-store-test-support"; inherit version; workDir = ./.; diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1459203104c6..bca404cde455 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,12 +1,8 @@ -#include // Needed by rapidcheck on Darwin -#include +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include -#include "nix/store/path-regex.hh" -#include "nix/store/store-api.hh" - #include "nix/util/tests/hash.hh" #include "nix/store/tests/path.hh" @@ -20,7 +16,6 @@ void showValue(const StorePath & p, std::ostream & os) } // namespace nix namespace rc { -using namespace nix; Gen storePathChar() { @@ -52,18 +47,19 @@ Gen storePathChar() gen::inRange(0, 10 + 2 * 26 + 6)); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( + return gen::construct( gen::suchThat(gen::container(storePathChar()), [](const std::string & s) { return !(s == "" || s == "." || s == ".." || s.starts_with(".-") || s.starts_with("..-")); })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( - gen::arbitrary(), gen::apply([](StorePathName n) { return n.name; }, gen::arbitrary())); + return gen::construct( + gen::arbitrary(), + gen::apply([](nix::StorePathName n) { return n.name; }, gen::arbitrary())); } } // namespace rc diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index b7e8f83f9e7c..a1d8ddee6412 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -75,25 +75,13 @@ INSTANTIATE_TEST_SUITE_P( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index d63cbeeeed2c..5afde2bdf7e2 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -1,12 +1,9 @@ -#include - #include #include #include "nix/util/json-utils.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/tests/protocol.hh" #include "nix/util/tests/characterization.hh" @@ -108,69 +105,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json index 4baadb547758..ec3479d5564b 100644 --- a/src/libstore-tests/data/build-result/success.json +++ b/src/libstore-tests/data/build-result/success.json @@ -1,14 +1,10 @@ { "builtOutputs": { "bar": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "signatures": [] }, "foo": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/common-protocol/realisation.bin and b/src/libstore-tests/data/common-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/common-protocol/realisation.json +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json new file mode 100644 index 000000000000..44006012fb8c --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json @@ -0,0 +1,31 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json new file mode 100644 index 000000000000..0706e1b64aa8 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json @@ -0,0 +1,27 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json new file mode 100644 index 000000000000..2bfe56fc8d44 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json @@ -0,0 +1,28 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json new file mode 100644 index 000000000000..453efae619de --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json @@ -0,0 +1,19 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json new file mode 100644 index 000000000000..7bc4cad36c51 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json @@ -0,0 +1,23 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json new file mode 100644 index 000000000000..3e92638f9d6f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json @@ -0,0 +1,34 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo\"]}}" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json new file mode 100644 index 000000000000..3f15525837fa --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json new file mode 100644 index 000000000000..b3970f4a3788 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json @@ -0,0 +1,20 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json new file mode 100644 index 000000000000..d9befe3c96e7 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json @@ -0,0 +1,26 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo\"]}}" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json new file mode 100644 index 000000000000..ed9b9bc48a56 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json @@ -0,0 +1,17 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": [], + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json new file mode 100644 index 000000000000..8ade9a6d90a9 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json @@ -0,0 +1,20 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json new file mode 100644 index 000000000000..cb57937ff53b --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json @@ -0,0 +1,25 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": {}, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "resolution-failure", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json new file mode 100644 index 000000000000..5c318ff84084 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json @@ -0,0 +1,33 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/nix/store/j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev", + "DEP1_OUT": "prefix-/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out-suffix", + "DEP2": "/nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out", + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "nested": { + "dep2": "before /nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json new file mode 100644 index 000000000000..6b39e5fc067f --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json @@ -0,0 +1,46 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/1r6jj1sjvjx4d1jqlnak71d8vzysdl2gyz2m01spjf31nhr1g55f", + "DEP1_OUT": "prefix-/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55-suffix", + "DEP2": "/1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55", + "nested": { + "dep2": "before /1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json new file mode 100644 index 000000000000..a15a69869a30 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json @@ -0,0 +1,23 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "dev" + }, + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + ], + [ + { + "drvPath": "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv", + "output": "out" + }, + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + ] +] diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index 804bbf07da6f..f7563abe6a68 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json index b5c8b8c5621d..576de0838177 100644 --- a/src/libstore-tests/data/dummy-store/one-realisation.json +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -1,8 +1,7 @@ { "buildTrace": { - "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": { "out": { - "dependentRealisations": {}, "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/nar-info/json-3/impure.json b/src/libstore-tests/data/nar-info/json-3/impure.json new file mode 100644 index 000000000000..eb6d4d48b83b --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/impure.json @@ -0,0 +1,31 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "compression": "xz", + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 3 +} diff --git a/src/libstore-tests/data/nar-info/json-3/pure.json b/src/libstore-tests/data/nar-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_impure.json b/src/libstore-tests/data/path-info/json-3/empty_impure.json new file mode 100644 index 000000000000..47d5030318b3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_impure.json @@ -0,0 +1,12 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_pure.json b/src/libstore-tests/data/path-info/json-3/empty_pure.json new file mode 100644 index 000000000000..6cc032782a4b --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_pure.json @@ -0,0 +1,8 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/impure.json b/src/libstore-tests/data/path-info/json-3/impure.json new file mode 100644 index 000000000000..f7ec907dcef3 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/impure.json @@ -0,0 +1,27 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/pure.json b/src/libstore-tests/data/path-info/json-3/pure.json new file mode 100644 index 000000000000..32fc916489e1 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/realisation/simple-fingerprint.txt b/src/libstore-tests/data/realisation/simple-fingerprint.txt new file mode 100644 index 000000000000..c9700924c28d --- /dev/null +++ b/src/libstore-tests/data/realisation/simple-fingerprint.txt @@ -0,0 +1 @@ +{"key":{"drvPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv","outputName":"foo"},"value":{"outPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}} \ No newline at end of file diff --git a/src/libstore-tests/data/realisation/simple-sig.json b/src/libstore-tests/data/realisation/simple-sig.json new file mode 100644 index 000000000000..084dbf0ba7ee --- /dev/null +++ b/src/libstore-tests/data/realisation/simple-sig.json @@ -0,0 +1,4 @@ +{ + "keyName": "test-key", + "sig": "WsjxK4/EI74COtzMDI+VCjQU9O4FydK0+YeY1CE5hUevogd+T+CPvNXza7oog3GTMS+ZlBwsC2S3ppwusKnJDg==" +} diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721198..1e4760b56827 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/unkeyed-simple.json b/src/libstore-tests/data/realisation/unkeyed-simple.json new file mode 100644 index 000000000000..2cf61d215206 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-simple.json @@ -0,0 +1,4 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/unkeyed-with-signature.json b/src/libstore-tests/data/realisation/unkeyed-with-signature.json new file mode 100644 index 000000000000..c51c6127a765 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-with-signature.json @@ -0,0 +1,9 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json deleted file mode 100644 index a58e0d7fe1c2..000000000000 --- a/src/libstore-tests/data/realisation/with-dependent-realisations.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependentRealisations": { - "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" - }, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] -} diff --git a/src/libstore-tests/data/realisation/with-signature-fingerprint.txt b/src/libstore-tests/data/realisation/with-signature-fingerprint.txt new file mode 100644 index 000000000000..bc67f80df0ef --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-fingerprint.txt @@ -0,0 +1 @@ +{"key":{"drvPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv","outputName":"foo"},"value":{"outPath":"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}} \ No newline at end of file diff --git a/src/libstore-tests/data/realisation/with-signature-sig.json b/src/libstore-tests/data/realisation/with-signature-sig.json new file mode 100644 index 000000000000..41c243b61ba4 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-sig.json @@ -0,0 +1,4 @@ +{ + "keyName": "test-key", + "sig": "ZE30UrOh26/EYvUaczjJtz6c4WcRYke6IEhDNN5TiFRF6Uu+H8lHofponlhfZxDDrCUlfstS/vtv3Xm2F6M/AA==" +} diff --git a/src/libstore-tests/data/realisation/with-signature-unstructured.json b/src/libstore-tests/data/realisation/with-signature-unstructured.json new file mode 100644 index 000000000000..7952e43697b0 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-unstructured.json @@ -0,0 +1,12 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index 3270f1cdebf4..a9efc97ca9e6 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,15 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin new file mode 100644 index 000000000000..fd745215c033 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin differ diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 000000000000..fa072599578a Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.json b/src/libstore-tests/data/serve-protocol/build-result-2.8.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.8.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 000000000000..5be0b15a3456 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.json b/src/libstore-tests/data/serve-protocol/drv-output-2.8.json new file mode 100644 index 000000000000..6afbf20e83a8 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/drv-output-2.8.json @@ -0,0 +1,10 @@ +[ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "quux" + } +] diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin new file mode 100644 index 000000000000..7d1fd462f998 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json new file mode 100644 index 000000000000..d6dde7327670 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -0,0 +1,19 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/serve-protocol/realisation.bin and b/src/libstore-tests/data/serve-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/serve-protocol/realisation.json +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 000000000000..fce3a25dd881 Binary files /dev/null and b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin differ diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json new file mode 100644 index 000000000000..9c8277922c5a --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -0,0 +1,13 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json index 0f593f4248d1..55bbd81bf14a 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 9c1fa3134d49..5c0f436e1f60 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -11,7 +11,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -27,11 +27,17 @@ ], "registrationTime": null, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin new file mode 100644 index 000000000000..fd745215c033 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin differ diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json new file mode 100644 index 000000000000..9c97678dadad --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..11bec3e6e3b7 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..1d566b13bb1a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json @@ -0,0 +1,39 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..5be0b15a3456 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..6afbf20e83a8 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json @@ -0,0 +1,10 @@ +[ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "quux" + } +] diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..7d1fd462f998 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..d6dde7327670 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -0,0 +1,19 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 44885cbdafda..cbd942ed477a 100644 Binary files a/src/libstore-tests/data/worker-protocol/realisation.bin and b/src/libstore-tests/data/worker-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json index 034d620306f0..ad1ae3ab5f4e 100644 --- a/src/libstore-tests/data/worker-protocol/realisation.json +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin new file mode 100644 index 000000000000..fce3a25dd881 Binary files /dev/null and b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin differ diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json new file mode 100644 index 000000000000..9c8277922c5a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -0,0 +1,13 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json index 9cc53c6804e9..fb2fbeef0c1d 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json index 427c286ddfbc..e62fd7a236bb 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -26,6 +26,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index 2c377145a10c..8bffd4410f6c 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": true, - "version": 2 + "version": 3 }, { "ca": null, @@ -24,12 +24,18 @@ ], "registrationTime": 23423, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -48,6 +54,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 7f0f62f876c0..ce95439d6e01 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } @@ -31,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index 93f4fb22a620..c596ae7f88fa 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } @@ -31,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index 617984a27eb6..ff2d1a627b84 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -1,11 +1,16 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "out": { + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } } }, "config": { @@ -31,7 +36,29 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index a63d6243fae3..61f4d8a760a2 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -1,15 +1,13 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } }, - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", "signatures": [] } @@ -38,7 +36,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { @@ -60,9 +58,30 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, - "derivations": {} + "derivations": { + "am42vl3rp8w29f5hyp9ljckxll50icpl-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [ + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + ] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } } diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json index f22d4c7dfbf1..55e16a6d38f8 100644 --- a/src/libstore-tests/data/worker-substitution/single/substituter.json +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json index 3f2994dfb455..765340031047 100644 --- a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { @@ -47,7 +47,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 16988613c633..bb7ad28f3d78 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -1,14 +1,11 @@ #include -#include -#include "nix/util/experimental-features.hh" #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" -#include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" @@ -307,7 +304,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) }; DerivationOptions advancedAttributes_structuredAttrs_defaults = { - .outputChecks = std::map::OutputChecks>{}, + .outputChecks = std::map::OutputChecks, std::less<>>{}, .unsafeDiscardReferences = {}, .passAsFile = {}, .exportReferencesGraph = {}, @@ -352,7 +349,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { DerivationOptions expected = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"dev", DerivationOptions::OutputChecks{ .maxSize = 789, @@ -384,7 +381,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { // Delete all keys but "dev" in options.outputChecks auto * outputChecksMapP = - std::get_if::OutputChecks>>( + std::get_if::OutputChecks, std::less<>>>( &options.outputChecks); ASSERT_TRUE(outputChecksMapP); auto & outputChecksMap = *outputChecksMapP; @@ -405,7 +402,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ia = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{pathFoo}, @@ -447,7 +444,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ca = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{placeholderFoo}, @@ -489,24 +486,137 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; -#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ - } \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ +#define TEST_JSON_OPTIONS(FIXUTURE, INPUT, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_from_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_to_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ } -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ca) + +/** + * `DerivationOptions` versions of the IA fixtures, used to + * exercise the `StorePath` JSON serializers. The IA test data files are + * reused as-is (see comment near the test invocations). + */ +static const StorePath spFoo{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + spFooDev{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, spBar{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + spBarDev{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, spBarDrv{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}; + +static const DerivationOptions advancedAttributes_sp_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_all_set = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = std::set>{spFoo}, + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + .disallowedRequisites = std::set>{spBarDev}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_defaults = { + .outputChecks = std::map::OutputChecks, std::less<>>{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_all_set = { + .outputChecks = + std::map::OutputChecks, std::less<>>{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = std::set>{spFoo}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .disallowedRequisites = std::set>{spBarDev}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +/** + * Same JSON characterization tests, but for `DerivationOptions`. + * + * Since `DrvRef` and `DrvRef` (when only + * the `Opaque` case is used) JSON-encode identically, the IA test data + * files can be reused as-is. + */ +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, defaults, sp_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, all_set, sp_all_set) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_defaults, sp_structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_all_set, sp_structuredAttrs_all_set) #undef TEST_JSON_OPTIONS diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index f0aa721cb7a9..5bc4cec9f38f 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -1,13 +1,12 @@ #include #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" -#include "nix/util/experimental-features.hh" #include "nix/util/tests/test-data.hh" #include "nix/store/store-open.hh" #include #include -using namespace nix; +namespace nix { // Benchmark parsing real derivation files static void BM_ParseRealDerivationFile(benchmark::State & state, const std::string & filename) @@ -54,3 +53,5 @@ BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "deriv BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); + +} // namespace nix diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index a9be99f996ed..e31b3e85442f 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -15,6 +15,14 @@ TEST_F(DerivationTest, BadATerm_version) parseDerivation(*store, readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); } +TEST_F(DerivationTest, UnterminatedString) +{ + ASSERT_THROW( + parseDerivation( + *store, "Derive([(\"out\",\"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo", "bar", mockXpSettings), + FormatError); +} + TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( @@ -177,44 +185,41 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values(makeSimpleDrv())); +INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = + { + { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = + { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = StringPairs{ + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; + }())); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index 115d5bc4bb1f..e825655d66f4 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -6,8 +6,6 @@ #include "nix/store/dummy-store-impl.hh" #include "nix/util/tests/json-characterization.hh" -#include "derivation/test-support.hh" - namespace nix { class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTest diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc new file mode 100644 index 000000000000..60b86f571205 --- /dev/null +++ b/src/libstore-tests/derivations.cc @@ -0,0 +1,365 @@ +#include +#include + +#include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" +#include "nix/store/downstream-placeholder.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class TryResolveTest : public LibStoreTest, public CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "try-resolve"; + +public: + + /** + * A simple in-memory *derived* build trace. (e.g. we have + * `SingleDerivedPath::Built` not `StorePath` keys, because that is + * how the `tryResolved` callback works.) + * + * For a real-world build trace, derived build traces are much more + * work to get correct, and should be at most a caching layer atop + * an underlying source-of-truth build trace. (This is as described + * in the manual.) + * + * However, this is just for some simple in-memory unit tests, with + * tiny amounts of data we can hand-review, so it's fine. The point + * is unit testing `tryResolve`, not unit testing "deep queries" + * from a non-derived build trace (what is done in + * `outputs-query.cc`) anyways --- that would be a separate unit + * test. + */ + struct BuildTrace + { + std::map dict; + + bool operator==(const BuildTrace &) const = default; + }; + +protected: + + EnableExperimentalFeature caFeature{"ca-derivations"}; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + /** + * Just here because we do this a few times in a tests. + */ + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a callback from a BuildTrace lookup. + */ + static auto makeCallback(const BuildTrace & table) + { + return + [&table](ref drvPath, const std::string & outputName) -> std::optional { + if (auto p = get(table.dict, SingleDerivedPath::Built{drvPath, outputName})) + return *p; + return std::nullopt; + }; + } + + /** + * Helper for the `exportReferencesGraph` + placeholder subpath tests. + */ + void exportRefGraphSubpathTest(std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed); + + /** + * Checkpoint before/buildTrace/after and assert the resolved derivation + * matches expected. + */ + void resolveExpect( + std::string_view stem, const Derivation & drv, const BuildTrace & buildTrace, const BasicDerivation & expected) + { + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-after", *resolved); + + EXPECT_EQ(*resolved, expected); + } +}; + +} // namespace nix + +JSON_IMPL(nix::TryResolveTest::BuildTrace); + +namespace nlohmann { + +using nix::SingleDerivedPath; +using nix::StorePath; +using nix::TryResolveTest; + +void adl_serializer::to_json(json & j, const TryResolveTest::BuildTrace & t) +{ + j = t.dict; +} + +TryResolveTest::BuildTrace adl_serializer::from_json(const json & j) +{ + return TryResolveTest::BuildTrace{ + .dict = j.get>(), + }; +} + +} // namespace nlohmann + +namespace nix { + +TEST_F(TryResolveTest, noInputs) +{ + resolveExpect( + "no-inputs", + [&] { + Derivation drv; + drv.name = "no-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.env = {{"FOO", "bar"}}; + return drv; + }(), + {}, + [&] { + BasicDerivation expected; + expected.name = "no-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = {{"out", caFloatingOutput()}}; + expected.env = {{"FOO", "bar"}}; + return expected; + }()); +} + +TEST_F(TryResolveTest, withInputs) +{ + // dep1 has two outputs (out, dev), dep2 has one (out) + StorePath dep1DrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv"}; + StorePath dep1OutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out"}; + StorePath dep1DevPath{"j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev"}; + StorePath dep2DrvPath{"h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv"}; + StorePath dep2OutPath{"i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out"}; + + auto placeholder1Out = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "out").render(); + auto placeholder1Dev = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "dev").render(); + auto placeholder2Out = DownstreamPlaceholder::unknownCaOutput(dep2DrvPath, "out").render(); + + DerivationOutputs multiOutputs = { + {"out", caFloatingOutput()}, + {"dev", caFloatingOutput()}, + }; + + resolveExpect( + "with-inputs", + [&] { + Derivation drv; + drv.name = "with-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = multiOutputs; + drv.inputDrvs = { + .map = { + {dep1DrvPath, {.value = {"out", "dev"}}}, + {dep2DrvPath, {.value = {"out"}}}, + }}; + drv.env = { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }; + drv.structuredAttrs = StructuredAttrs{{ + {"dep1out", placeholder1Out}, + {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, + }}; + return drv; + }(), + {.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "out", + }, + dep1OutPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "dev", + }, + dep1DevPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep2DrvPath), + .output = "out", + }, + dep2OutPath, + }, + }}, + [&] { + BasicDerivation expected; + expected.name = "with-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = multiOutputs; + expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.env = { + {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, + {"DEP1_DEV", store->printStorePath(dep1DevPath)}, + {"DEP2", store->printStorePath(dep2OutPath)}, + }; + expected.structuredAttrs = StructuredAttrs{{ + {"dep1out", store->printStorePath(dep1OutPath)}, + {"nested", + nlohmann::json::object({{"dep2", "before " + store->printStorePath(dep2OutPath) + " after"}})}, + }}; + return expected; + }()); +} + +TEST_F(TryResolveTest, resolutionFailure) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + + Derivation drv; + drv.name = "resolution-failure"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + BuildTrace buildTrace; + + checkpointJson(*this, "resolution-failure-before", drv); + checkpointJson(*this, "resolution-failure-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + EXPECT_FALSE(resolved); +} + +/** + * Test that `derivationOptionsFromStructuredAttrs` can parse + * `exportReferencesGraph` entries that reference a CA derivation output + * with a subpath appended (e.g. `${dep}/foo`), and that the parsed + * options resolve correctly. + * + * Regression test for #15003: the placeholder + subpath was not found + * in the placeholder map (exact-match only), and the fallback to + * `toStorePath()` failed because placeholders don't start with + * `/nix/store/`. + */ +void TryResolveTest::exportRefGraphSubpathTest( + std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + StorePath depOutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out"}; + + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + + nix::checkpointJson(*this, std::string{stem} + "-before-options", options); + + SingleDerivedPath expectedPath = SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }; + + ASSERT_EQ(options.exportReferencesGraph.size(), 1); + auto it = options.exportReferencesGraph.find("refs"); + ASSERT_NE(it, options.exportReferencesGraph.end()); + ASSERT_EQ(it->second.size(), 1); + EXPECT_EQ(*it->second.begin(), expectedPath); + + // Also test that resolution works + BuildTrace buildTrace{.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }, + depOutPath, + }, + }}; + + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-resolved", *resolved); + + // Re-parse options from the resolved derivation, where placeholders + // have been substituted with concrete store paths. + auto resolvedOptions = derivationOptionsFromStructuredAttrs( + *store, + /* inputDrvs */ {}, + resolved->env, + resolved->structuredAttrs ? &*resolved->structuredAttrs : nullptr, + true); + + nix::checkpointJson(*this, std::string{stem} + "-resolved-options", resolvedOptions); + + EXPECT_EQ( + resolvedOptions.exportReferencesGraph, + (decltype(resolvedOptions.exportReferencesGraph){ + {"refs", std::set{SingleDerivedPath::Opaque{depOutPath}}}})); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.env = { + {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + }; + + exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath-sa"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.structuredAttrs = StructuredAttrs{{ + {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, + }}; + drv.env = { + {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, + }; + + exportRefGraphSubpathTest("export-ref-subpath-sa", drv, &*drv.structuredAttrs); +} + +} // namespace nix diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 70e789c0c4fd..46297c9a30db 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -1,5 +1,3 @@ -#include - #include #include @@ -94,8 +92,6 @@ TEST_F(DerivedPathTest, built_built_xp) MissingExperimentalFeature); } -#ifndef COVERAGE - /* TODO: Disabled due to the following error: path '00000000000000000000000000000000-0^0' is not a valid store path: @@ -115,8 +111,6 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store), xpSettings)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index e3422fe585f8..4626fc31f28d 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -6,6 +6,7 @@ #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { @@ -27,30 +28,67 @@ class DummyStoreTest : public virtual CharacterizationTest } }; +TEST(DummyStore, storeDir_default) +{ + DummyStoreConfig config{{}}; + EXPECT_EQ(config.storeDir, "/nix/store"); +} + +TEST(DummyStore, storeDir_absolutePath) +{ + DummyStoreConfig config{{{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(DummyStore, storeDir_canonicalized) +{ + DummyStoreConfig config{{{"store", "/my//store/../store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(DummyStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW((DummyStoreConfig{{{"store", "my/store"}}}), UsageError); +} + +TEST(DummyStore, storeDir_empty_rejected) +{ + EXPECT_THROW((DummyStoreConfig{{{"store", ""}}}), UsageError); +} + +TEST(DummyStore, getStateDir_default) +{ + // DummyStore uses the base StoreConfig::getStateDir which returns + // the global settings.nixStateDir + DummyStoreConfig config{{}}; + EXPECT_EQ(config.getStateDir(), settings.nixStateDir); +} + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); + EnableExperimentalFeature enableCA{"ca-derivations"}; + auto store = [] { auto cfg = make_ref(StoreReference::Params{}); cfg->readOnly = false; return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, value}}}); + store->buildTrace.insert({drvPath, {{outputName, value}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); @@ -131,10 +169,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { [&] { auto store = writeCfg->openDummyStore(); store->buildTrace.insert_or_assign( - Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, std::map{ { "out", diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 000000000000..b89bb7ed0bc1 --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/filetransfer-retry.cc b/src/libstore-tests/filetransfer-retry.cc new file mode 100644 index 000000000000..4cd1d2dd0cd6 --- /dev/null +++ b/src/libstore-tests/filetransfer-retry.cc @@ -0,0 +1,155 @@ +#include + +#include + +#include "nix/store/filetransfer-impl.hh" + +namespace nix { + +// --------------------------------------------------------------------------- +// Deterministic (jitter=false) — table-driven +// --------------------------------------------------------------------------- + +struct RetryDelayCase +{ + std::string description; + uint32_t attempt; + uint32_t baseMs; + uint32_t ceilMs; + std::optional retryAfterMs; + long long expectedMs; // std::chrono::milliseconds::rep +}; + +class RetryDelayDeterministicTest : public ::testing::TestWithParam +{}; + +TEST_P(RetryDelayDeterministicTest, yieldsExpectedDelay) +{ + auto & p = GetParam(); + std::mt19937 rng{0}; + auto result = computeRetryDelayMs( + {.attempt = p.attempt, .baseMs = p.baseMs, .ceilMs = p.ceilMs, .retryAfterMs = p.retryAfterMs, .jitter = false}, + rng); + EXPECT_EQ(result.count(), p.expectedMs); +} + +INSTANTIATE_TEST_SUITE_P( + computeRetryDelayMs, + RetryDelayDeterministicTest, + ::testing::Values( + // --- exponential growth --- + RetryDelayCase{"exponential_attempt1", 1, 100, 60000, {}, 100}, + RetryDelayCase{"exponential_attempt2", 2, 100, 60000, {}, 200}, + RetryDelayCase{"exponential_attempt3", 3, 100, 60000, {}, 400}, + RetryDelayCase{"exponential_attempt4", 4, 100, 60000, {}, 800}, + RetryDelayCase{"exponential_attempt5", 5, 100, 60000, {}, 1600}, + + // --- ceiling --- + RetryDelayCase{"ceil_at_5000", 10, 100, 5000, {}, 5000}, // 100*2^9=51200 > 5000 + RetryDelayCase{"base_exceeds_ceil", 1, 10000, 5000, {}, 5000}, // base > ceil on attempt 1 + RetryDelayCase{"base_equals_ceil_a1", 1, 500, 500, {}, 500}, + RetryDelayCase{"base_equals_ceil_a5", 5, 500, 500, {}, 500}, + RetryDelayCase{"base_equals_ceil_a10", 10, 500, 500, {}, 500}, + RetryDelayCase{"ceil_zero_a1", 1, 100, 0, {}, 0}, + RetryDelayCase{"ceil_zero_a5", 5, 100, 0, {}, 0}, + + // --- Retry-After interaction --- + RetryDelayCase{"retry_after_exceeds_computed", 1, 100, 60000, 3000, 3000}, + RetryDelayCase{"retry_after_under_computed", 4, 100, 60000, 500, 800}, + RetryDelayCase{"retry_after_above_ceil", 1, 100, 60000, 120000, 120000}, + RetryDelayCase{"retry_after_zero", 1, 100, 60000, 0, 100}, + RetryDelayCase{"retry_after_equals_ceil", 1, 100, 60000, 60000, 60000}, + RetryDelayCase{"retry_after_equals_computed", 3, 100, 60000, 400, 400}, // max(400,400)=400 + + // --- rate-limit base (429/503 path) --- + RetryDelayCase{"rate_limit_a1", 1, 5000, 60000, {}, 5000}, + RetryDelayCase{"rate_limit_a2", 2, 5000, 60000, {}, 10000}, + RetryDelayCase{"rate_limit_a3", 3, 5000, 60000, {}, 20000}, + RetryDelayCase{"rate_limit_a4", 4, 5000, 60000, {}, 40000}, + RetryDelayCase{"rate_limit_a5", 5, 5000, 60000, {}, 60000}, // ceiled + RetryDelayCase{"rate_limit_retry_after_under", 2, 5000, 60000, 3000, 10000}, + + // --- attempt boundaries --- + RetryDelayCase{"attempt_zero", 0, 100, 60000, {}, 100}, // shift=0, same as attempt 1 + RetryDelayCase{"attempt_one", 1, 100, 60000, {}, 100}, + + // --- shift clamp boundary --- + // base=1: 1<<31 = 2147483648, ceiled at UINT32_MAX + RetryDelayCase{"shift_clamp_at_32", 32, 1, UINT32_MAX, {}, 2147483648LL}, + // attempt 33 → shift still 31 (clamped), same result + RetryDelayCase{"shift_clamp_at_33", 33, 1, UINT32_MAX, {}, 2147483648LL}, + + // --- zero / minimal base --- + RetryDelayCase{"zero_base", 1, 0, 60000, {}, 0}, + RetryDelayCase{"base_one_a1", 1, 1, 60000, {}, 1}, + RetryDelayCase{"base_one_a2", 2, 1, 60000, {}, 2}, + RetryDelayCase{"base_one_a11", 11, 1, 60000, {}, 1024}, + + // --- integer extremes --- + RetryDelayCase{"uint_max_base_small_ceil", 1, UINT32_MAX, 1000, {}, 1000}, + RetryDelayCase{"uint_max_ceil", 1, 100, UINT32_MAX, {}, 100}, + + // --- overflow guard --- + RetryDelayCase{"overflow_attempt100", 100, 1000, 60000, {}, 60000}, + RetryDelayCase{"overflow_attempt1M", 1000000, 100, 60000, {}, 60000}), + [](const auto & info) { return info.param.description; }); + +// --------------------------------------------------------------------------- +// Jitter — loop-based, kept as individual TEST()s +// --------------------------------------------------------------------------- + +TEST(computeRetryDelayMs, jitter_stays_in_bounds) +{ + std::mt19937 rng{42}; + for (int i = 0; i < 1000; i++) { + auto ms = computeRetryDelayMs({.attempt = 3, .baseMs = 100, .ceilMs = 60000, .jitter = true}, rng).count(); + EXPECT_LE(ms, 400); // 100 * 2^2 = 400 + } +} + +TEST(computeRetryDelayMs, jitter_with_retry_after_floor) +{ + // computed = 100, Retry-After = 5000 → jitter in [5000, 5100] + std::mt19937 rng{42}; + for (int i = 0; i < 1000; i++) { + auto ms = computeRetryDelayMs( + {.attempt = 1, .baseMs = 100, .ceilMs = 60000, .retryAfterMs = 5000, .jitter = true}, rng) + .count(); + EXPECT_GE(ms, 5000); + EXPECT_LE(ms, 5100); + } +} + +TEST(computeRetryDelayMs, zero_base_jitter_returns_zero) +{ + std::mt19937 rng{0}; + // ceiled == 0 → early return, no distribution created + EXPECT_EQ(computeRetryDelayMs({.attempt = 1, .baseMs = 0, .ceilMs = 60000, .jitter = true}, rng).count(), 0); +} + +TEST(computeRetryDelayMs, jitter_with_ceil_zero) +{ + std::mt19937 rng{42}; + // ceilMs=0 → backoff=0 → ceiling<=floor early return, rng untouched + EXPECT_EQ(computeRetryDelayMs({.attempt = 1, .baseMs = 100, .ceilMs = 0, .jitter = true}, rng).count(), 0); +} + +TEST(computeRetryDelayMs, jitter_ceiled_one) +{ + // Smallest non-trivial jitter range: [0, 1] + std::mt19937 rng{42}; + bool sawZero = false, sawOne = false; + for (int i = 0; i < 100; i++) { + auto ms = computeRetryDelayMs({.attempt = 1, .baseMs = 1, .ceilMs = 1, .jitter = true}, rng).count(); + EXPECT_LE(ms, 1); + EXPECT_GE(ms, 0); + if (ms == 0) + sawZero = true; + if (ms == 1) + sawOne = true; + } + EXPECT_TRUE(sawZero); + EXPECT_TRUE(sawOne); +} + +} // namespace nix diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 8495db4b5abb..74f3b93cd3df 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -3,12 +3,22 @@ #include "nix/store/http-binary-cache-store.hh" #include "nix/store/tests/https-store.hh" -#include "nix/util/fs-sink.hh" namespace nix { using Authority = ParsedURL::Authority; +TEST(HttpBinaryCacheStore, storeDir_absolutePath) +{ + HttpBinaryCacheStoreConfig config{parseURL("https://example.com"), {{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(HttpBinaryCacheStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(HttpBinaryCacheStoreConfig(parseURL("https://example.com"), {{"store", "my/store"}}), UsageError); +} + TEST(HttpBinaryCacheStore, constructConfig) { HttpBinaryCacheStoreConfig config{ diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index 35543cf40064..926f4dc38741 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -4,6 +4,20 @@ namespace nix { +TEST(LegacySSHStore, storeDir_absolutePath) +{ + LegacySSHStoreConfig config{ + ParsedURL::Authority::parse("localhost"), + {{"store", "/my/store"}}, + }; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(LegacySSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(LegacySSHStoreConfig(ParsedURL::Authority::parse("localhost"), {{"store", "my/store"}}), UsageError); +} + TEST(LegacySSHStore, constructConfig) { LegacySSHStoreConfig config( diff --git a/src/libstore-tests/local-binary-cache-store.cc b/src/libstore-tests/local-binary-cache-store.cc index 295976488b5f..9ae1519a6fc1 100644 --- a/src/libstore-tests/local-binary-cache-store.cc +++ b/src/libstore-tests/local-binary-cache-store.cc @@ -4,6 +4,18 @@ namespace nix { +TEST(LocalBinaryCacheStore, storeDir_absolutePath) +{ + LocalBinaryCacheStoreConfig config{std::filesystem::path("/foo/bar/baz"), {{"store", "/my/store"}}}; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(LocalBinaryCacheStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW( + LocalBinaryCacheStoreConfig(std::filesystem::path("/foo/bar/baz"), {{"store", "my/store"}}), UsageError); +} + TEST(LocalBinaryCacheStore, constructConfig) { LocalBinaryCacheStoreConfig config{std::filesystem::path("/foo/bar/baz"), {}}; diff --git a/src/libstore-tests/local-fs-store.cc b/src/libstore-tests/local-fs-store.cc new file mode 100644 index 000000000000..22f7ca4084f4 --- /dev/null +++ b/src/libstore-tests/local-fs-store.cc @@ -0,0 +1,72 @@ +#include + +#include "nix/store/local-fs-store.hh" +#include "nix/store/globals.hh" + +namespace nix { + +namespace { + +/** + * Concrete subclass of `LocalFSStoreConfig` for testing, since + * `LocalFSStoreConfig` is abstract (`openStore()` is pure virtual). + */ +struct TestLocalFSStoreConfig : LocalFSStoreConfig +{ + TestLocalFSStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Native) + , LocalFSStoreConfig(params) + { + } + + TestLocalFSStoreConfig(const std::filesystem::path & path, const Params & params) + : StoreConfig(params, FilePathType::Native) + , LocalFSStoreConfig(path, params) + { + } + + ref openStore() const override + { + unreachable(); + } +}; + +} // namespace + +TEST(LocalFSStoreConfig, getStateDir_default) +{ + TestLocalFSStoreConfig config{{}}; + // Default stateDir should be the global settings value + EXPECT_EQ(config.getStateDir(), settings.nixStateDir); +} + +TEST(LocalFSStoreConfig, getStateDir_withRoot) +{ + std::filesystem::path root = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + root /= "foo"; + root /= "bar"; + auto expectedStateDir = root / "nix" / "var" / "nix"; + TestLocalFSStoreConfig config{root, {}}; + EXPECT_EQ(config.getStateDir(), expectedStateDir); +} + +TEST(LocalFSStoreConfig, getStateDir_explicitSetting) +{ + std::filesystem::path stateDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + stateDir /= "custom"; + stateDir /= "state"; + TestLocalFSStoreConfig config{{{"state", stateDir.string()}}}; + EXPECT_EQ(config.getStateDir(), stateDir); +} + +} // namespace nix diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index c207564255f3..eb88d879ce20 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -6,24 +6,52 @@ namespace nix { TEST(LocalOverlayStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalOverlayStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalOverlayStore, constructConfig_rootPath) { - LocalOverlayStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalOverlayStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); +} + +TEST(LocalOverlayStore, upperLayer_notOverridden) +{ + LocalOverlayStoreConfig config{"", {}}; + EXPECT_FALSE(config.upperLayer.isOverridden()); +} + +TEST(LocalOverlayStore, upperLayer_overridden) +{ + LocalOverlayStoreConfig config{ + "", + { + {"upper-layer", "/some/upper"}, + }, + }; + EXPECT_TRUE(config.upperLayer.isOverridden()); + EXPECT_EQ(config.upperLayer.get(), std::filesystem::path{"/some/upper"}); } } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index 554d42efc811..06da1b008e07 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -10,26 +10,60 @@ namespace nix { +TEST(LocalStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + LocalStoreConfig config{"", {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(LocalStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(LocalStoreConfig("", {{"store", (std::filesystem::path{"nix"} / "store").string()}}), UsageError); +} + +TEST(LocalStore, storeDir_empty_rejected) +{ + EXPECT_THROW(LocalStoreConfig("", {{"store", ""}}), UsageError); +} + TEST(LocalStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_rootPath) { - LocalStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_to_string) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index f0a334c1b613..7c9a17ed6b00 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -2,18 +2,18 @@ #include "nix/util/file-system.hh" #include "nix/util/util.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/test-data.hh" #include #include -using testing::Contains; -using testing::ElementsAre; -using testing::Eq; -using testing::Field; -using testing::SizeIs; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Field; +using ::testing::SizeIs; -using namespace nix; +namespace nix { TEST(machines, getMachinesWithEmptyBuilders) { @@ -27,7 +27,7 @@ TEST(machines, getMachinesUriOnly) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -49,7 +49,7 @@ TEST(machines, getMachinesDefaults) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -185,7 +185,10 @@ TEST(machines, getMachinesWithCorrectFileReference) TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - std::filesystem::path path = "/dev/null"; + auto tmpDir = nix::createTempDir(); + AutoDelete delTmpDir(tmpDir); + auto path = tmpDir / "empty-machines"; + nix::writeFile(path, ""); ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); @@ -207,3 +210,5 @@ TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } + +} // namespace nix diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc index c45e3a7f384a..93d93e4aa0ce 100644 --- a/src/libstore-tests/main.cc +++ b/src/libstore-tests/main.cc @@ -3,11 +3,9 @@ #include "nix/store/tests/test-main.hh" #include "nix/store/tests/libstore-network.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 2d12b14d3cb5..a7353f1ec7dd 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -60,12 +60,16 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation/external-formats.cc', 'derivation/invariants.cc', + 'derivations.cc', 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', + 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', + 'local-fs-store.cc', 'local-overlay-store.cc', 'local-store.cc', 'machines.cc', @@ -73,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', @@ -82,6 +87,7 @@ sources = files( 's3-url.cc', 'serve-protocol.cc', 'ssh-store.cc', + 'store-open.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol.cc', @@ -105,14 +111,19 @@ this_exe = executable( cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) +test_env = { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', +} + +if host_machine.system() != 'windows' + test_env += {'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store'} +endif + test( meson.project_name(), this_exe, - env : { - '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'HOME' : meson.current_build_dir() / 'test-home', - 'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store', - }, + env : test_env, protocol : 'gtest', ) diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index aebefc775675..b54c98fe5e7d 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -15,6 +15,13 @@ TEST(NarInfoDiskCacheImpl, create_and_read) int prio = 12345; bool wantMassQuery = true; + auto mkFields = [](bool wantMassQuery, int prio) { + return std::map{ + {"WantMassQuery", wantMassQuery ? "1" : "0"}, + {"Priority", std::to_string(prio)}, + }; + }; + auto tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir); auto dbPath(tmpDir / "test-narinfo-disk-cache.sqlite"); @@ -30,21 +37,19 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // Set up "background noise" and check that different caches receive different ids { - auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); - auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + auto bc1 = cache->createCache("https://bar", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.fields = mkFields(false, 12)}); ASSERT_NE(bc1, bc2); barId = bc1; } // Check that the fields are saved and returned correctly. This does not test // the select statement yet, because of in-memory caching. - savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); - ; + savedId = cache->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(savedId, r->id); } @@ -67,8 +72,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read) { auto r = cache->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); } } @@ -84,13 +88,12 @@ TEST(NarInfoDiskCacheImpl, create_and_read) } // "Update", same data, check that the id number is reused - cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + cache2->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)}); { auto r = cache2->upToDateCacheExists("http://foo"); ASSERT_TRUE(r); - ASSERT_EQ(r->priority, prio); - ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio)); ASSERT_EQ(r->id, savedId); } @@ -107,10 +110,9 @@ TEST(NarInfoDiskCacheImpl, create_and_read) auto r0 = cache2->upToDateCacheExists("https://bar"); ASSERT_FALSE(r0); - cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + cache2->createCache("https://bar", "/nix/storedir", {.fields = mkFields(!wantMassQuery, prio + 10)}); auto r = cache2->upToDateCacheExists("https://bar"); - ASSERT_EQ(r->wantMassQuery, !wantMassQuery); - ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->fields, mkFields(!wantMassQuery, prio + 10)); ASSERT_EQ(r->id, barId); } diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9b0f6018cdeb..63e5908d8818 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -31,6 +31,16 @@ class NarInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class NarInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { auto info = NarInfo::makeFromCA( @@ -65,7 +75,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; - info.compression = "xz"; + info.compression = CompressionAlgo::xz; info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); info.fileSize = 4029176; } @@ -122,6 +132,31 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) JSON_READ_TEST_V2(STEM, PURE) \ JSON_WRITE_TEST_V2(STEM, PURE) +#define JSON_READ_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(nullptr, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, PURE) \ + JSON_READ_TEST_V3(STEM, PURE) \ + JSON_WRITE_TEST_V3(STEM, PURE) + JSON_TEST_V1(pure, false) JSON_TEST_V1(impure, true) @@ -131,4 +166,19 @@ JSON_READ_TEST_V1(pure_noversion, false) JSON_TEST_V2(pure, false) JSON_TEST_V2(impure, true) +JSON_TEST_V3(pure, false) +JSON_TEST_V3(impure, true) + +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + } // namespace nix diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 9bfbf8453a9e..0c4739bbc91f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -5,6 +5,7 @@ #include "nix_api_util.h" #include "nix_api_store.h" +#include "nix/store/tests/libstore.hh" #include "nix/store/tests/nix_api_store.hh" #include "nix/store/globals.hh" #include "nix/util/tests/string_callback.hh" @@ -76,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } @@ -212,6 +213,9 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine/Windows does not support symlinks needed for local store gcroots"; +#endif auto tmp = nix::createTempDir(); auto storeRoot = (tmp / "store").string(); auto stateDir = (tmp / "state").string(); @@ -251,8 +255,9 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - Store * store = - nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir().string()).c_str(), nullptr); + auto tmpDir = nix::createTempDir() / "binary-cache"; + auto url = nix::ParsedURL{.scheme = "file", .path = nix::pathToUrlPath(tmpDir)}; + Store * store = nix_store_open(ctx, url.to_string().c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); @@ -289,6 +294,7 @@ struct LambdaAdapter class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base { public: + std::optional enableCA; StorePath * drvPath = nullptr; nix_derivation * drv = nullptr; Store * store = nullptr; @@ -297,8 +303,11 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base void SetUp() override { nix_api_store_test_base::SetUp(); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + enableCA.emplace("ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; store = open_local_store(); @@ -352,8 +361,11 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base TEST_F(nix_api_store_test_base, build_from_json) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // FIXME get rid of these - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -399,8 +411,11 @@ TEST_F(nix_api_store_test_base, build_from_json) TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the system is invalid - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -444,8 +459,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the builder fails - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -489,8 +507,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when builder succeeds but produces no output - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -686,7 +707,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) // Test that nix_store_realise returns outputs in alphabetical order by output name. // This test uses a CA derivation with 10 outputs in randomized input order // to verify that the callback order is deterministic and alphabetical. - nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::EnableExperimentalFeature enableCA{"ca-derivations"}; nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -956,4 +977,55 @@ TEST_F(nix_api_store_test, nix_derivation_clone) nix_derivation_free(drv2); } +TEST_F(nix_api_store_test, nix_store_build_paths) +{ + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.getWorkerSettings().substituters = {}; + + auto * store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + // Replace the hardcoded system with the current system + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get()); + + auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + // Realise the derivation - capture the order outputs are returned + std::map outputs; + std::vector output_order; + auto cb = LambdaAdapter{.fun = [&](const char * path, const char * result) { + ASSERT_NE(path, nullptr); + ASSERT_NE(result, nullptr); + output_order.push_back(path); + outputs.emplace(path, result); + }}; + + std::vector paths = {drvPath}; + + auto ret = nix_store_build_paths( + ctx, + store, + const_cast(paths.data()), + paths.size(), + decltype(cb)::call_void, + static_cast(&cb)); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + ASSERT_EQ(outputs.size(), 1); + + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + } // namespace nixC diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 000000000000..4be25e2ad3b8 --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + Derivation drv; + drv.name = std::move(name); + drv.platform = "x86_64-linux"; + drv.builder = "/bin/sh"; + drv.outputs = {{"out", caFloatingOutput()}}; + return drv; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index 1fac222fccba..ae0bb1789b2b 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -33,7 +33,16 @@ class ExtendedOutputsSpecTest : public virtual CharacterizationTest TEST_F(OutputsSpecTest, no_empty_names) { +#ifndef _WIN32 ASSERT_DEATH(OutputsSpec::Names{StringSet{}}, ""); +#else + // ASSERT_DEATH relies on CreateProcess, which under Wine leaks the + // child's stderr into the parent console. Test both sides of the + // invariant without a death test instead. + ASSERT_FALSE(OutputsSpec::parseOpt("")); + OutputsSpec::Names names{StringSet{"out"}}; + ASSERT_FALSE(names.empty()); +#endif } #define TEST_DONT_PARSE(NAME, STR) \ @@ -259,13 +268,9 @@ INSTANTIATE_TEST_SUITE_P( #undef TEST_JSON -#ifndef COVERAGE - RC_GTEST_PROP(OutputsSpec, prop_round_rip, (const OutputsSpec & o)) { RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } -#endif - } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 97ad4b270ade..7734ee175561 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -30,6 +30,16 @@ class PathInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class PathInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static UnkeyedValidPathInfo makeEmpty() { return { @@ -129,6 +139,31 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo JSON_READ_TEST_V2(STEM, OBJ) \ JSON_WRITE_TEST_V2(STEM, OBJ, PURE) +#define JSON_READ_TEST_V3(STEM, OBJ) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(nullptr, encoded); \ + auto expected = OBJ; \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, OBJ, PURE) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return OBJ.toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, OBJ, PURE) \ + JSON_READ_TEST_V3(STEM, OBJ) \ + JSON_WRITE_TEST_V3(STEM, OBJ, PURE) + JSON_TEST_V1(empty_pure, makeEmpty(), false) JSON_TEST_V1(empty_impure, makeEmpty(), true) JSON_TEST_V1(pure, makeFull(*store, false), false) @@ -142,6 +177,23 @@ JSON_TEST_V2(empty_impure, makeEmpty(), true) JSON_TEST_V2(pure, makeFull(*store, false), false) JSON_TEST_V2(impure, makeFull(*store, true), true) +JSON_TEST_V3(empty_pure, makeEmpty(), false) +JSON_TEST_V3(empty_impure, makeEmpty(), true) +JSON_TEST_V3(pure, makeFull(*store, false), false) +JSON_TEST_V3(impure, makeFull(*store, true), true) + +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index eb860a34dab1..65755b7f9645 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -86,8 +86,6 @@ TEST_DO_PARSE(triple_dot, "...") #undef TEST_DO_PARSE -#ifndef COVERAGE - RC_GTEST_FIXTURE_PROP(StorePathTest, prop_regex_accept, (const StorePath & p)) { RC_ASSERT(std::regex_match(std::string{p.name()}, nameRegex)); @@ -141,8 +139,6 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) RC_ASSERT(parsed == std::regex_match(std::string{name}, nameRegex)); } -#endif - /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d2d7df80f59a..cec9db9087ec 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -1,16 +1,57 @@ -#include - #include #include #include #include "nix/store/store-api.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/signature/local-keys.hh" +#include "nix/util/signature/signer.hh" #include "nix/util/tests/json-characterization.hh" #include "nix/store/tests/libstore.hh" namespace nix { +using nlohmann::json; + +/* ---------------------------------------------------------------------------- + * Test data + * --------------------------------------------------------------------------*/ + +UnkeyedRealisation unkeyedSimple{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, +}; + +UnkeyedRealisation unkeyedWithSignature{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, +}; + +DrvOutput testDrvOutput{ + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", +}; + +auto testSecretKey = + "test-key:tU7tTvLcScf8pmz/eTV0BEtLmRsPpZfKaRcd0nCN+pysBZPHSeg61/u2oc7mIOewfuAY1V1BiX32homTaDJ2Jw=="; + +Realisation simple{ + unkeyedSimple, + testDrvOutput, +}; + +Realisation withSignature{ + unkeyedWithSignature, + testDrvOutput, +}; + +/* ---------------------------------------------------------------------------- + * Realisation JSON + * --------------------------------------------------------------------------*/ + class RealisationTest : public JsonCharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "realisation"; @@ -23,12 +64,6 @@ class RealisationTest : public JsonCharacterizationTest, public Lib } }; -/* ---------------------------------------------------------------------------- - * JSON - * --------------------------------------------------------------------------*/ - -using nlohmann::json; - struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface> {}; @@ -44,19 +79,6 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, -}; - INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -67,24 +89,159 @@ INSTANTIATE_TEST_SUITE_P( }, std::pair{ "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }; - return r; - }(), + withSignature, })); /** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. + * Old signature format (string) should still be parseable. */ -TEST_F(RealisationTest, dependent_realisations_from_json) +TEST_F(RealisationTest, with_signature_from_json) +{ + readJsonTest("with-signature-unstructured", withSignature); +} + +/* ---------------------------------------------------------------------------- + * UnkeyedRealisation JSON + * --------------------------------------------------------------------------*/ + +class UnkeyedRealisationTest : public JsonCharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "realisation"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +struct UnkeyedRealisationJsonTest : UnkeyedRealisationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(UnkeyedRealisationJsonTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(UnkeyedRealisationJsonTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + UnkeyedRealisationJSON, + UnkeyedRealisationJsonTest, + ::testing::Values( + std::pair{ + "unkeyed-simple", + unkeyedSimple, + }, + std::pair{ + "unkeyed-with-signature", + unkeyedWithSignature, + })); + +/* ---------------------------------------------------------------------------- + * Signing and verification + * --------------------------------------------------------------------------*/ + +struct RealisationSigningTest : RealisationTest, ::testing::WithParamInterface> +{}; + +TEST_P(RealisationSigningTest, fingerprint) +{ + const auto & [name, realisation] = GetParam(); + writeTest(std::string{name} + "-fingerprint.txt", [&]() -> std::string { + return realisation.fingerprint(realisation.id); + }); +} + +TEST_P(RealisationSigningTest, sign) { - readJsonTest("with-dependent-realisations", simple); + const auto & [name, realisation] = GetParam(); + + LocalSigner signer(SecretKey::parse(testSecretKey)); + + auto sig = realisation.sign(realisation.id, signer); + + nix::writeJsonTest(*this, std::string{name} + "-sig", sig); } +TEST_P(RealisationSigningTest, verify) +{ + const auto & [name, realisation] = GetParam(); + + auto publicKey = SecretKey::parse(testSecretKey)->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(publicKey->name, std::move(publicKey)); + + readTest(std::string{name} + "-sig.json", [&](const auto & encoded) { + Signature sig = json::parse(encoded); + ASSERT_TRUE(realisation.checkSignature(realisation.id, publicKeys, sig)); + }); +} + +TEST_P(RealisationSigningTest, verify_rejects_wrong_key) +{ + const auto & [name, realisation] = GetParam(); + + auto wrongKey = SecretKey::generate("wrong-key", KeyType::Ed25519); + auto wrongPublicKey = wrongKey->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(wrongPublicKey->name, std::move(wrongPublicKey)); + + auto r = static_cast(realisation); + LocalSigner signer(SecretKey::parse(testSecretKey)); + r.sign(realisation.id, signer); + + ASSERT_EQ(r.checkSignatures(realisation.id, publicKeys), 0); +} + +TEST_P(RealisationSigningTest, verify_rejects_tampered_outpath) +{ + const auto & [name, realisation] = GetParam(); + + auto publicKey = SecretKey::parse(testSecretKey)->toPublicKey(); + PublicKeys publicKeys; + publicKeys.insert_or_assign(publicKey->name, std::move(publicKey)); + + auto r = static_cast(realisation); + LocalSigner signer(SecretKey::parse(testSecretKey)); + r.sign(realisation.id, signer); + + // Tamper with the output path after signing. + r.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}; + + ASSERT_EQ(r.checkSignatures(realisation.id, publicKeys), 0); +} + +TEST_P(RealisationSigningTest, signatures_stripped_from_fingerprint) +{ + const auto & [name, realisation] = GetParam(); + + auto fp = realisation.fingerprint(realisation.id); + auto parsed = json::parse(fp); + auto obj = getObject(parsed); + auto * value = optionalValueAt(obj, "value"); + ASSERT_NE(value, nullptr); + ASSERT_FALSE(getObject(*value).contains("signatures")); +} + +INSTANTIATE_TEST_SUITE_P( + RealisationSigning, + RealisationSigningTest, + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + withSignature, + })); + } // namespace nix diff --git a/src/libstore-tests/ref-scan-bench.cc b/src/libstore-tests/ref-scan-bench.cc index ff0aa181503a..05264fd29061 100644 --- a/src/libstore-tests/ref-scan-bench.cc +++ b/src/libstore-tests/ref-scan-bench.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { template static void randomReference(std::mt19937 & urng, OIt outIter) @@ -91,3 +91,5 @@ static void BM_RefScanSinkRandom(benchmark::State & state) } BENCHMARK(BM_RefScanSinkRandom)->Arg(10'000)->Arg(100'000)->Arg(1'000'000)->Arg(5'000'000)->Arg(10'000'000); + +} // namespace nix diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index f2c6fb51e5ca..69b89b1460ee 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -4,6 +4,9 @@ #include +#include +#include + namespace nix { struct RewriteParams @@ -43,6 +46,100 @@ INSTANTIATE_TEST_CASE_P( RewriteParams{"foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, RewriteParams{"foooo", "foooo", {}})); +TEST(references, rewritingSinkChunking) +{ + std::mt19937 rng(42); + + /* Build a set of rewrites. Keys are random [a-z] strings of random + length in [8, 32]; values are same-length [A-Z] strings so a + replacement never produces a new match for any key. We also skip + any key that would be a substring of (or have as a substring) an + existing key, so that inserting one key into the input cannot also + produce a match for a different key. */ + StringMap rewrites; + std::vector keys; + { + std::uniform_int_distribution lowerDist('a', 'z'); + std::uniform_int_distribution upperDist('A', 'Z'); + std::uniform_int_distribution lenDist(8, 32); + + while (rewrites.size() < 8) { + std::string from(lenDist(rng), '\0'); + for (auto & c : from) + c = lowerDist(rng); + if (rewrites.count(from)) + continue; + bool overlap = false; + for (auto & other : keys) + if (from.find(other) != std::string::npos || other.find(from) != std::string::npos) { + overlap = true; + break; + } + if (overlap) + continue; + std::string to(from.size(), '\0'); + for (auto & c : to) + c = upperDist(rng); + rewrites[from] = to; + keys.push_back(from); + } + } + + /* Build a ~1 MB input mixing rewrite keys with [0-9] digits. Always + emit at least one digit between two consecutive keys so adjacency + cannot create spurious matches at key boundaries. Compute the + expected output string and matches vector at the same time. */ + std::string input; + std::string expectedOutput; + std::set expectedMatches; + input.reserve(1'000'000); + expectedOutput.reserve(1'000'000); + { + std::uniform_int_distribution digitDist('0', '9'); + std::uniform_int_distribution keyDist(0, keys.size() - 1); + std::bernoulli_distribution useKeyDist(0.2); + bool justInsertedKey = false; + + while (input.size() < 1'000'000) { + if (useKeyDist(rng) && !justInsertedKey) { + const auto & from = keys[keyDist(rng)]; + expectedMatches.insert(input.size()); + expectedOutput += rewrites.at(from); + input += from; + justInsertedKey = true; + } else { + char d = static_cast(digitDist(rng)); + input.push_back(d); + expectedOutput.push_back(d); + justInsertedKey = false; + } + } + } + + StringSink singleOut; + RewritingSink singleSink(rewrites, singleOut); + singleSink(input); + singleSink.flush(); + + StringSink chunkedOut; + RewritingSink chunkedSink(rewrites, chunkedOut); + { + std::uniform_int_distribution chunkDist(1, 128); + std::string_view remaining(input); + while (!remaining.empty()) { + auto n = std::min(chunkDist(rng), remaining.size()); + chunkedSink(remaining.substr(0, n)); + remaining = remaining.substr(n); + } + } + chunkedSink.flush(); + + ASSERT_EQ(singleOut.s, expectedOutput); + ASSERT_EQ(chunkedOut.s, expectedOutput); + ASSERT_EQ(singleSink.matches, expectedMatches); + ASSERT_EQ(chunkedSink.matches, expectedMatches); +} + TEST(references, scan) { std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0"; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 1d795818369e..ecea1c8010a4 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -5,14 +5,13 @@ #include "nix/store/store-open.hh" #include "nix/util/file-system.hh" #include "nix/util/hash.hh" -#include "nix/util/tests/test-data.hh" #ifndef _WIN32 # include # include -using namespace nix; +namespace nix { static void BM_RegisterValidPathsDerivations(benchmark::State & state) { @@ -23,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); @@ -67,7 +66,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } @@ -76,4 +75,6 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10); +} // namespace nix + #endif diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 9aa9b2dd1a3e..126d7a522aaa 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -1,7 +1,6 @@ #include "nix/store/s3-binary-cache-store.hh" #include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" -#include "nix/store/s3-url.hh" #include diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 159abeb60827..e34f916cbb5c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -1,5 +1,4 @@ #include -#include #include #include @@ -17,8 +16,6 @@ namespace nix { const char serveProtoDir[] = "serve-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct ServeProtoTest : VersionedProtoTest { /** @@ -78,16 +75,19 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -96,54 +96,37 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -209,7 +192,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", @@ -242,27 +228,72 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_8, + "build-result-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -292,12 +323,12 @@ VERSIONED_CHARACTERIZATION_TEST( }), (std::tuple{ ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.narSize = 34878; info; }), ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.deriver = StorePath{ "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; @@ -322,7 +353,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -513,12 +544,6 @@ TEST_F(ServeProtoTest, handshake_log) }); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(ServeProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { @@ -578,4 +603,62 @@ TEST_F(ServeProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_7_compat, + "build-result-2.7-compat", + (ServeProto::Version{ + .major = 2, + .minor = 7, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index 2d4548d6ae21..813d58574ce5 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -6,6 +6,20 @@ namespace nix { +TEST(SSHStore, storeDir_absolutePath) +{ + SSHStoreConfig config{ + ParsedURL::Authority::parse("localhost"), + {{"store", "/my/store"}}, + }; + EXPECT_EQ(config.storeDir, "/my/store"); +} + +TEST(SSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(SSHStoreConfig(ParsedURL::Authority::parse("localhost"), {{"store", "my/store"}}), UsageError); +} + TEST(SSHStore, constructConfig) { SSHStoreConfig config{ @@ -31,6 +45,27 @@ TEST(SSHStore, constructConfig) EXPECT_EQ(config.getReference().render(/*withParams=*/true), "ssh-ng://me@localhost:2222"); } +TEST(MountedSSHStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + MountedSSHStoreConfig config{{.host = "localhost"}, {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(MountedSSHStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW( + MountedSSHStoreConfig({.host = "localhost"}, {{"store", (std::filesystem::path{"nix"} / "store").string()}}), + UsageError); +} + TEST(MountedSSHStore, constructConfig) { MountedSSHStoreConfig config{ diff --git a/src/libstore-tests/store-open.cc b/src/libstore-tests/store-open.cc new file mode 100644 index 000000000000..cadeb3f040a4 --- /dev/null +++ b/src/libstore-tests/store-open.cc @@ -0,0 +1,62 @@ +#include + +#include "nix/store/store-open.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/local-store.hh" +#include "nix/store/globals.hh" +#include "nix/util/file-system.hh" +#include "nix/util/finally.hh" + +namespace nix { + +TEST(StoreOpen, resolveStoreConfig_auto_default) +{ + // Save original settings + // + // TODO: resolveStoreConfig should not depend on global settings; + // the test should not have to override them. + auto originalStateDir = settings.nixStateDir; + Finally restoreStateDir([&]() { settings.nixStateDir = originalStateDir; }); + + // Set up a temporary writable state directory + auto tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + auto stateDir = tmpDir / "var/nix"; + createDirs(stateDir); + settings.nixStateDir = stateDir; + + StoreReference ref{ + .variant = StoreReference::Auto{}, + .params = {}, + }; + + auto config = resolveStoreConfig(std::move(ref)); + + // With a writable state directory and no daemon socket, "auto" should resolve to LocalStore + auto * localConfig = dynamic_cast(config.get()); + ASSERT_NE(localConfig, nullptr); + EXPECT_EQ(localConfig->getStateDir(), stateDir); +} + +TEST(StoreOpen, resolveStoreConfig_auto_withParams) +{ + // Create a temporary directory with a writable state directory + auto tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + auto stateDir = tmpDir / "var/nix"; + createDirs(stateDir); + + StoreReference ref{ + .variant = StoreReference::Auto{}, + .params = {{"state", stateDir.string()}}, + }; + + auto config = resolveStoreConfig(std::move(ref)); + + // With a writable state directory and no daemon socket, "auto" should resolve to LocalStore + auto * localConfig = dynamic_cast(config.get()); + ASSERT_NE(localConfig, nullptr); + EXPECT_EQ(localConfig->getStateDir(), stateDir); +} + +} // namespace nix diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index 88af22dbb22d..32122f5c2b86 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -4,6 +4,25 @@ namespace nix { +TEST(UDSRemoteStore, storeDir_absolutePath) +{ + std::filesystem::path storeDir = +#ifdef _WIN32 + "C:\\"; +#else + "/"; +#endif + storeDir /= "nix"; + storeDir /= "store"; + UDSRemoteStoreConfig config{"", {{"store", storeDir.string()}}}; + EXPECT_EQ(config.storeDir, storeDir.string()); +} + +TEST(UDSRemoteStore, storeDir_relativePath_rejected) +{ + EXPECT_THROW(UDSRemoteStoreConfig("", {{"store", (std::filesystem::path{"nix"} / "store").string()}}), UsageError); +} + TEST(UDSRemoteStore, constructConfig) { UDSRemoteStoreConfig config{"/tmp/socket", {}}; diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index aac5822d01f8..91acde7380f6 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -18,17 +17,17 @@ namespace nix { TEST(WorkerProtoVersionNumber, ordering) { using Number = WorkerProto::Version::Number; - EXPECT_LT((Number{1, 10}), (Number{1, 20})); - EXPECT_GT((Number{1, 30}), (Number{1, 20})); - EXPECT_EQ((Number{1, 10}), (Number{1, 10})); - EXPECT_LT((Number{0, 255}), (Number{1, 0})); + EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20})); + EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20})); + EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10})); + EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0})); } TEST(WorkerProtoVersion, partialOrderingSameFeatures) { using V = WorkerProto::Version; - V v1{.number = {1, 20}, .features = {"a", "b"}}; - V v2{.number = {1, 30}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; EXPECT_TRUE(v1 < v2); EXPECT_TRUE(v2 > v1); @@ -40,8 +39,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures) TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) { using V = WorkerProto::Version; - V fewer{.number = {1, 30}, .features = {"a"}}; - V more{.number = {1, 30}, .features = {"a", "b"}}; + V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; // fewer <= more: JUST the features are a subset EXPECT_TRUE(fewer < more); @@ -54,8 +53,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered) { using V = WorkerProto::Version; // Same number but incomparable features - V v1{.number = {1, 20}, .features = {"a", "c"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -69,8 +68,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) { using V = WorkerProto::Version; // Higher number but fewer features — unordered - V v1{.number = {1, 30}, .features = {"a"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -80,8 +79,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) { using V = WorkerProto::Version; - V empty{.number = {1, 20}, .features = {}}; - V some{.number = {1, 30}, .features = {"a"}}; + V empty{.number = {.major = 1, .minor = 20}, .features = {}}; + V some{.number = {.major = 1, .minor = 30}, .features = {"a"}}; // empty features is a subset of everything EXPECT_TRUE(empty < some); @@ -91,8 +90,6 @@ TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) const char workerProtoDir[] = "worker-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct WorkerProtoTest : VersionedProtoTest { /** @@ -223,69 +220,67 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + unkeyedRealisation_realisation_with_path, + "unkeyed-realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, - }, + .features = {"realisation-with-path-not-hash"}, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + realisation_realisation_with_path, + "realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, + .features = {"realisation-with-path-not-hash"}, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -318,7 +313,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", @@ -347,25 +345,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -374,7 +360,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", @@ -410,27 +397,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -443,7 +416,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", @@ -479,27 +453,71 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_realisation_with_path, + "build-result-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -576,7 +594,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.registrationTime = 23423; @@ -585,7 +603,7 @@ VERSIONED_CHARACTERIZATION_TEST( }), ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -620,7 +638,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -634,7 +652,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -675,7 +693,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -690,7 +708,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -920,7 +938,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 123}, + .number = + { + .major = 1, + .minor = 123, + }, .features = {"bar", "aap", "mies", "xyzzy"}, }); }); @@ -931,7 +953,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 200}, + .number = + { + .major = 1, + .minor = 200, + }, .features = {"foo", "bar", "xyzzy"}, }); @@ -950,12 +976,6 @@ TEST_F(WorkerProtoTest, handshake_features) })); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(WorkerProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { @@ -1013,4 +1033,65 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29_compat, + "build-result-1.29-compat", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3a049b7e7216..3534d44d0d8f 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -176,8 +176,7 @@ TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject) TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) { - // Enable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + EnableExperimentalFeature enableCA{"ca-derivations"}; // Create a CA floating output derivation Derivation drv; @@ -198,12 +197,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Snapshot the destination store before checkpointJson("ca-drv/store-before", dummyStore); - // Compute the hash modulo of the derivation - // For CA floating derivations, the kind is Deferred since outputs aren't known until build - auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); - ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); - auto drvHash = hashModulo.hashes.at("out"); - // Create the output store object auto outputPath = substituter->addToStore( "test-ca-drv-out", @@ -222,7 +215,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Add the realisation (build trace) to the substituter substituter->buildTrace.insert_or_assign( - drvHash, + drvPath, std::map{ { "out", @@ -236,7 +229,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) checkpointJson("ca-drv/substituter", substituter); // The realisation should not exist in the destination store yet - DrvOutput drvOutput{drvHash, "out"}; + DrvOutput drvOutput{drvPath, "out"}; ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter @@ -248,7 +241,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Create a derivation goal for the CA derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + auto goal = worker.makeDerivationGoal(drvPath, make_ref(drv), "out", bmNormal, true); // Run the worker Goals goals; @@ -268,9 +261,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Verify the goal succeeded ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); - - // Disable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", ""); } /** @@ -280,8 +270,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) */ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) { - // Enable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + EnableExperimentalFeature enableCA{"ca-derivations"}; // Create the dependency CA floating derivation Derivation depDrv; @@ -299,11 +288,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the dependency derivation to the destination store auto depDrvPath = dummyStore->writeDerivation(depDrv); - // Compute the hash modulo for the dependency derivation - auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); - ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); - auto depDrvHash = depHashModulo.hashes.at("out"); - // Create the output store object for the dependency in the substituter auto depOutputPath = substituter->addToStore( "dep-drv-out", @@ -322,7 +306,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Add the realisation for the dependency to the substituter substituter->buildTrace.insert_or_assign( - depDrvHash, + depDrvPath, std::map{ { "out", @@ -350,14 +334,22 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the root derivation to the destination store auto rootDrvPath = dummyStore->writeDerivation(rootDrv); + // Resolve the root derivation using a callback that returns the dep output path + auto resolvedRootDrv = rootDrv.tryResolve( + *substituter, + [&](ref drvPath, const std::string & outputName) -> std::optional { + EXPECT_EQ(*drvPath, SingleDerivedPath::Opaque{depDrvPath}); + EXPECT_EQ(outputName, "out"); + return depOutputPath; + }); + ASSERT_TRUE(resolvedRootDrv); + + // Write the resolved derivation to the substituter + auto resolvedRootDrvPath = substituter->writeDerivation(Derivation{*resolvedRootDrv}); + // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); - // Compute the hash modulo for the root derivation - auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); - ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); - auto rootDrvHash = rootHashModulo.hashes.at("out"); - // Create the output store object for the root derivation // Note: it does NOT reference the dependency's output auto rootOutputPath = substituter->addToStore( @@ -378,12 +370,13 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) HashAlgorithm::SHA256); // The DrvOutputs for both derivations - DrvOutput depDrvOutput{depDrvHash, "out"}; - DrvOutput rootDrvOutput{rootDrvHash, "out"}; + DrvOutput depDrvOutput{depDrvPath, "out"}; + DrvOutput resolvedRootDrvOutput{resolvedRootDrvPath, "out"}; - // Add the realisation for the root derivation to the substituter + // Add the realisation for the *resolved* root derivation to the substituter + // (not the original root derivation - that would be an illegal "deep" realisation) substituter->buildTrace.insert_or_assign( - rootDrvHash, + resolvedRootDrvPath, std::map{ { "out", @@ -394,12 +387,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) }); // Snapshot the substituter - // Note: it has realisations for both drvs, but only the root's output store object + // Note: it has the dep realisation, resolved root drv, resolved root realisation, and root output checkpointJson("issue-11928/substituter", substituter); // The realisations should not exist in the destination store yet ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput)); - ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput)); + ASSERT_FALSE(dummyStore->queryRealisation(resolvedRootDrvOutput)); // Create a worker with our custom substituter Worker worker{*dummyStore, *dummyStore}; @@ -410,7 +403,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Create a derivation goal for the root derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + auto goal = worker.makeDerivationGoal(rootDrvPath, make_ref(rootDrv), "out", bmNormal, false); // Run the worker Goals goals; @@ -423,28 +416,22 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // The root output path should now exist in the destination store ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath)); - // The root realisation should now exist in the destination store - auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput); + // The resolved root realisation should now exist in the destination store + auto rootRealisation = dummyStore->queryRealisation(resolvedRootDrvOutput); ASSERT_TRUE(rootRealisation); ASSERT_EQ(rootRealisation->outPath, rootOutputPath); - // #11928: The dependency's REALISATION should be fetched, because - // it is needed to resolve the underlying derivation. Currently the - // realisation is not fetched (bug). Once fixed: Change - // depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the - // ASSERT_EQ + // The dependency's REALISATION should have been fetched auto depRealisation = dummyStore->queryRealisation(depDrvOutput); - ASSERT_FALSE(depRealisation); - // ASSERT_EQ(depRealisation->outPath, depOutputPath); + ASSERT_TRUE(depRealisation); + ASSERT_EQ(depRealisation->outPath, depOutputPath); - // The dependency's OUTPUT is correctly not fetched (not referenced by root output) - ASSERT_FALSE(dummyStore->isValidPath(depOutputPath)); + // TODO #11928: The dependency's OUTPUT should NOT be fetched (not referenced + // by root output). Once #11928 is fixed, change ASSERT_TRUE to ASSERT_FALSE. + ASSERT_TRUE(dummyStore->isValidPath(depOutputPath)); // Verify the goal succeeded ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); - - // Disable CA derivations experimental feature - experimentalFeatureSettings.set("extra-experimental-features", ""); } } // namespace nix diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c68753823d19..43060cb697b8 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -27,22 +27,19 @@ class WriteDerivationTest : public LibStoreTest ref config; }; -static Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; -} - } // namespace TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = makeSimpleDrv(); + auto drv = []() { + Derivation drv; + drv.name = "simple-derivation"; + drv.platform = "system"; + drv.builder = "foo"; + drv.args = {"bar", "baz"}; + drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; + return drv; + }(); auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/active-builds.cc b/src/libstore/active-builds.cc new file mode 100644 index 000000000000..4b2688a9dd25 --- /dev/null +++ b/src/libstore/active-builds.cc @@ -0,0 +1,156 @@ +#include "nix/store/active-builds.hh" +#include "nix/util/json-utils.hh" + +#include + +#ifndef _WIN32 +# include +#endif + +namespace nix { + +void QueryActiveBuildsStore::anchor() {} + +void TrackActiveBuildsStore::anchor() {} + +UserInfo UserInfo::fromUid(uid_t uid) +{ + UserInfo info; + info.uid = uid; + +#ifndef _WIN32 + // Look up the user name for the UID (thread-safe) + struct passwd pwd; + struct passwd * result; + std::vector buf(16384); + if (getpwuid_r(uid, &pwd, buf.data(), buf.size(), &result) == 0 && result) + info.name = result->pw_name; +#endif + + return info; +} + +} // namespace nix + +namespace nlohmann { + +using namespace nix; + +UserInfo adl_serializer::from_json(const json & j) +{ + return UserInfo{ + .uid = j.at("uid").get(), + .name = j.contains("name") && !j.at("name").is_null() + ? std::optional(j.at("name").get()) + : std::nullopt, + }; +} + +void adl_serializer::to_json(json & j, const UserInfo & info) +{ + j = nlohmann::json{ + {"uid", info.uid}, + {"name", info.name}, + }; +} + +// Durations are serialized as floats representing seconds. +static std::optional parseDuration(const json & j, const char * key) +{ + if (j.contains(key) && !j.at(key).is_null()) + return std::chrono::duration_cast( + std::chrono::duration(j.at(key).get())); + else + return std::nullopt; +} + +static nlohmann::json printDuration(const std::optional & duration) +{ + return duration + ? nlohmann::json( + std::chrono::duration_cast>(*duration) + .count()) + : nullptr; +} + +ActiveBuildInfo::ProcessInfo adl_serializer::from_json(const json & j) +{ + return ActiveBuildInfo::ProcessInfo{ + .pid = j.at("pid").get(), + .parentPid = j.at("parentPid").get(), + .user = j.at("user").get(), + .argv = j.at("argv").get>(), + .utime = parseDuration(j, "utime"), + .stime = parseDuration(j, "stime"), + .cutime = parseDuration(j, "cutime"), + .cstime = parseDuration(j, "cstime"), + }; +} + +void adl_serializer::to_json(json & j, const ActiveBuildInfo::ProcessInfo & process) +{ + j = nlohmann::json{ + {"pid", process.pid}, + {"parentPid", process.parentPid}, + {"user", process.user}, + {"argv", process.argv}, + {"utime", printDuration(process.utime)}, + {"stime", printDuration(process.stime)}, + {"cutime", printDuration(process.cutime)}, + {"cstime", printDuration(process.cstime)}, + }; +} + +ActiveBuild adl_serializer::from_json(const json & j) +{ + auto type = j.at("type").get(); + if (type != "build") + throw Error("invalid active build JSON: expected type 'build' but got '%s'", type); + std::optional cgroup; + if (!j.at("cgroup").is_null()) + cgroup = j.at("cgroup").get(); + return ActiveBuild{ + .nixPid = j.at("nixPid").get(), + .clientPid = j.at("clientPid").get>(), + .clientUid = j.at("clientUid").get>(), + .mainPid = j.at("mainPid").get(), + .mainUser = j.at("mainUser").get(), + .cgroup = std::move(cgroup), + .startTime = (time_t) j.at("startTime").get(), + .derivation = StorePath{getString(j.at("derivation"))}, + }; +} + +void adl_serializer::to_json(json & j, const ActiveBuild & build) +{ + j = nlohmann::json{ + {"type", "build"}, + {"nixPid", build.nixPid}, + {"clientPid", build.clientPid}, + {"clientUid", build.clientUid}, + {"mainPid", build.mainPid}, + {"mainUser", build.mainUser}, + {"cgroup", build.cgroup ? nlohmann::json(*build.cgroup) : nlohmann::json(nullptr)}, + {"startTime", (double) build.startTime}, + {"derivation", build.derivation.to_string()}, + }; +} + +ActiveBuildInfo adl_serializer::from_json(const json & j) +{ + ActiveBuildInfo info(adl_serializer::from_json(j)); + info.processes = j.at("processes").get>(); + info.utime = parseDuration(j, "utime"); + info.stime = parseDuration(j, "stime"); + return info; +} + +void adl_serializer::to_json(json & j, const ActiveBuildInfo & build) +{ + adl_serializer::to_json(j, build); + j["processes"] = build.processes; + j["utime"] = printDuration(build.utime); + j["stime"] = printDuration(build.stime); +} + +} // namespace nlohmann diff --git a/src/libstore/async-path-writer.cc b/src/libstore/async-path-writer.cc new file mode 100644 index 000000000000..f3271093a3c1 --- /dev/null +++ b/src/libstore/async-path-writer.cc @@ -0,0 +1,186 @@ +#include "nix/store/async-path-writer.hh" +#include "nix/util/archive.hh" +#include "nix/util/provenance.hh" + +#include +#include + +namespace nix { + +struct AsyncPathWriterImpl : AsyncPathWriter +{ + ref store; + + struct Item + { + StorePath storePath; + std::string contents; + std::string name; + Hash hash; + StorePathSet references; + RepairFlag repair; + std::shared_ptr provenance; + std::promise promise; + }; + + struct State + { + std::vector items; + std::unordered_map> futures; + bool quit = false; + }; + + Sync state_; + + std::thread workerThread; + + std::condition_variable wakeupCV; + + AsyncPathWriterImpl(ref store) + : store(store) + { + workerThread = std::thread([&]() { + while (true) { + std::vector items; + + { + auto state(state_.lock()); + while (!state->quit && state->items.empty()) + state.wait(wakeupCV); + if (state->items.empty() && state->quit) + return; + std::swap(items, state->items); + } + + try { + writePaths(items); + for (auto & item : items) + item.promise.set_value(); + } catch (...) { + for (auto & item : items) + item.promise.set_exception(std::current_exception()); + } + } + }); + } + + virtual ~AsyncPathWriterImpl() + { + state_.lock()->quit = true; + wakeupCV.notify_all(); + workerThread.join(); + } + + StorePath addPath( + std::string contents, + std::string name, + StorePathSet references, + RepairFlag repair, + std::shared_ptr provenance) override + { + auto hash = hashString(HashAlgorithm::SHA256, contents); + + auto storePath = store->makeFixedOutputPathFromCA( + name, + TextInfo{ + .hash = hash, + .references = references, + }); + + auto state(state_.lock()); + std::promise promise; + state->futures.insert_or_assign(storePath, promise.get_future()); + state->items.push_back( + Item{ + .storePath = storePath, + .contents = std::move(contents), + .name = std::move(name), + .hash = hash, + .references = std::move(references), + .repair = repair, + .provenance = provenance, + .promise = std::move(promise), + }); + wakeupCV.notify_all(); + + return storePath; + } + + void waitForPath(const StorePath & path) override + { + auto future = ({ + auto state = state_.lock(); + auto i = state->futures.find(path); + if (i == state->futures.end()) + return; + i->second; + }); + future.get(); + } + + void waitForAllPaths() override + { + auto futures = ({ + auto state(state_.lock()); + std::move(state->futures); + }); + for (auto & future : futures) + future.second.get(); + } + + void writePaths(const std::vector & items) + { +// FIXME: addMultipeToStore() shouldn't require a NAR hash. +#if 0 + Store::PathsSource sources; + RepairFlag repair = NoRepair; + + for (auto & item : items) { + ValidPathInfo info{item.storePath, Hash(HashAlgorithm::SHA256)}; + info.references = item.references; + info.ca = ContentAddress { + .method = ContentAddressMethod::Raw::Text, + .hash = item.hash, + }; + if (item.repair) repair = item.repair; + auto source = sinkToSource([&](Sink & sink) + { + dumpString(item.contents, sink); + }); + sources.push_back({std::move(info), std::move(source)}); + } + + Activity act(*logger, lvlDebug, actUnknown, fmt("adding %d paths to the store", items.size())); + + store->addMultipleToStore(std::move(sources), act, repair); +#endif + + for (auto & item : items) { + StringSource source(item.contents); + store->addTempRoot(item.storePath); + auto storePath = store->addToStoreFromDump( + source, + item.storePath.name(), + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + item.references, + item.repair, + item.provenance); + assert(storePath == item.storePath); + } + } + + void anchor() override; +}; + +void AsyncPathWriter::anchor() {} + +void AsyncPathWriterImpl::anchor() {} + +ref AsyncPathWriter::make(ref store) +{ + return make_ref(store); +} + +} // namespace nix diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index b471962897bb..e4a909a4a554 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -4,6 +4,7 @@ # include # include "nix/store/s3-url.hh" +# include "nix/util/environment-variables.hh" # include "nix/util/logging.hh" # include @@ -27,12 +28,16 @@ namespace nix { +void AwsAuthError::anchor() {} + AwsAuthError::AwsAuthError(int errorCode) : CloneableError("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) , errorCode(errorCode) { } +AwsCredentialProvider::~AwsCredentialProvider() {} + namespace { /** @@ -170,15 +175,33 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Check whether the AWS SDK can resolve a region from the standard + * environment variables. Mirrors aws_credentials_provider_resolve_region_from_env. + */ +static bool awsRegionSetInEnv() +{ + return getEnvNonEmpty("AWS_REGION") || getEnvNonEmpty("AWS_DEFAULT_REGION"); +} + /** * Create an STS WebIdentity credentials provider using the C library directly. * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, * and AWS_REGION from the environment (falling back to the profile config). * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. * Returns nullptr if the required parameters can't be resolved. + * + * @param fallbackRegion Region to use when neither AWS_REGION nor + * AWS_DEFAULT_REGION is set — typically the ?region= from the S3 URL. + * Note: this also overrides any region from the profile config, since + * aws-c-auth gives options.region precedence over all implicit sources. + * Without this fallback the provider fails entirely in IRSA setups where + * the pod environment sets the token and role ARN but not the region + * (observed with AWS_CONFIG_FILE=/dev/null). */ static std::shared_ptr createSTSWebIdentityProvider( const std::string & profileName, + const std::string & fallbackRegion, Aws::Crt::Io::ClientBootstrap * bootstrap, Aws::Crt::Io::TlsContext * tlsContext, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) @@ -192,6 +215,12 @@ static std::shared_ptr createSTSWebIdentit options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); } + // aws-c-auth gives options.region precedence over env vars, so only set it + // when env vars are absent — otherwise we'd mask a user-supplied AWS_REGION. + if (!fallbackRegion.empty() && !awsRegionSetInEnv()) { + options.region = aws_byte_cursor_from_c_str(fallbackRegion.c_str()); + } + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); } @@ -259,8 +288,6 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr createProviderForProfile(const std::string & profile); + std::shared_ptr + createProviderForProfile(const std::string & profile, const std::string & region); private: Aws::Crt::ApiHandle apiHandle; std::shared_ptr tlsContext; Aws::Crt::Io::ClientBootstrap * bootstrap; - boost::concurrent_flat_map> - credentialProviderCache; + // Keyed by (profile, region). Region is part of the key because it is baked + // into the STS WebIdentity provider at construction time; two S3 URLs with + // different ?region= parameters need distinct chains. + boost:: + concurrent_flat_map, std::shared_ptr> + credentialProviderCache; }; std::shared_ptr -AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) +AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile, const std::string & region) { // profileDisplayName is only used for debug logging - SDK uses its default profile // when ProfileNameOverride is not set @@ -369,7 +402,7 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) bool ecsAdded = false; if (tlsContext) { addProviderToChain("STS WebIdentity", [&]() { - return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + return createSTSWebIdentityProvider(profile, region, bootstrap, tlsContext.get(), allocator); }); ecsAdded = addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); @@ -395,18 +428,19 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } -AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) +AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile, const std::string & region) { std::shared_ptr provider; + auto key = std::make_pair(profile, region); credentialProviderCache.try_emplace_and_cvisit( - profile, + key, nullptr, - [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); }, + [&](auto & kv) { provider = kv.second = createProviderForProfile(profile, region); }, [&](const auto & kv) { provider = kv.second; }); if (!provider) { - credentialProviderCache.erase_if(profile, [](const auto & kv) { + credentialProviderCache.erase_if(key, [](const auto & kv) { [[maybe_unused]] auto [_, provider] = kv; return !provider; }); @@ -419,6 +453,8 @@ AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & return getCredentialsFromProvider(provider); } +} // anonymous namespace + ref makeAwsCredentialsProvider() { return make_ref(); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 31fe2c6173da..14bf76b87e5e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -3,38 +3,44 @@ #include "nix/util/compression.hh" #include "nix/store/derivations.hh" #include "nix/util/source-accessor.hh" -#include "nix/store/globals.hh" +#include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info.hh" #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-info-disk-cache.hh" #include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/util/util.hh" #include #include #include -#include #include +#include #include namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + +void NoSuchBinaryCacheFile::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { - if (!config.secretKeyFile.get().empty()) - signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile.get())})); + if (auto & skf = config.secretKeyFile.get()) + signers.push_back(std::make_unique(SecretKey::parse(readFile(*skf)))); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); std::string keyPath; while (std::getline(ss, keyPath, ',')) { - signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); + signers.push_back(std::make_unique(SecretKey::parse(readFile(keyPath)))); } } @@ -43,8 +49,10 @@ BinaryCacheStore::BinaryCacheStore(Config & config) narMagic = sink.s; } -void BinaryCacheStore::init() +std::map BinaryCacheStore::parseNixCacheInfo() { + std::map fields; + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); @@ -62,13 +70,31 @@ void BinaryCacheStore::init() config.getHumanReadableURI(), value, storeDir); - } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); - } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); + } else { + /* Keep every other field verbatim, including ones we + don't (yet) understand. The known ones are applied + by applyCacheInfoFields(). */ + fields.insert_or_assign(name, value); } } } + + return fields; +} + +void BinaryCacheStore::applyCacheInfoFields(const std::map & fields) +{ + if (auto * value = get(fields, "WantMassQuery")) + config.wantMassQuery.setDefault(*value == "1"); + if (auto * value = get(fields, "Priority")) { + if (auto priority = string2Int(*value)) + config.priority.setDefault(*priority); + } +} + +void BinaryCacheStore::init() +{ + applyCacheInfoFields(parseNixCacheInfo()); } std::optional BinaryCacheStore::getNixCacheInfo() @@ -136,8 +162,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) std::shared_ptr(narInfo)); } -ref BinaryCacheStore::addToStoreCommon( - Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) +ref BinaryCacheStore::uploadData(Source & narSource, RepairFlag repair, fun mkInfo) { auto fdTemp = createAnonymousTempFile(); @@ -152,8 +177,10 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed{fileSink, fileHashSink}; - auto compressionSink = makeCompressionSink( - config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); + bool parallel = config.parallelCompression.overridden ? config.parallelCompression.get() + : config.compression.get() == CompressionAlgo::zstd; + auto compressionSink = + makeCompressionSink(config.compression, teeSinkCompressed, parallel, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; narAccessor = makeNarAccessor(parseNarListing(teeSource)); @@ -165,7 +192,7 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; @@ -187,19 +214,6 @@ ref BinaryCacheStore::addToStoreCommon( ((1.0 - (double) fileSize / info.narSize) * 100.0), duration); - /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) - try { - if (ref != info.path) - queryPathInfo(ref); - } catch (InvalidPath &) { - throw Error( - "cannot add '%s' to the binary cache because the reference '%s' is not valid", - printStorePath(info.path), - printStorePath(ref)); - } - /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (config.writeNARListing) { @@ -280,24 +294,45 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressedBytes += fileSize; stats.narWriteCompressionTimeMs += duration; + return narInfo; +} + +void BinaryCacheStore::uploadNarInfo(ref narInfo) +{ + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : narInfo->references) + try { + if (ref != narInfo->path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error( + "cannot add '%s' to the binary cache because the reference '%s' is not valid", + printStorePath(narInfo->path), + printStorePath(ref)); + } + narInfo->sign(*this, signers); /* Atomically write the NAR info file.*/ writeNarInfo(narInfo); stats.narInfoWrite++; +} +ref BinaryCacheStore::addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) +{ + auto narInfo = uploadData(narSource, repair, std::move(mkInfo)); + uploadNarInfo(narInfo); return narInfo; } void BinaryCacheStore::addToStore( const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) { - if (!repair && isValidPath(info.path)) { - // FIXME: copyNAR -> null sink - narSource.drain(); + if (!repair && isValidPath(info.path)) return; - } addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { /* FIXME reinstate these, once we can correctly do hash modulo sink as @@ -308,6 +343,137 @@ void BinaryCacheStore::addToStore( }}); } +void BinaryCacheStore::addMultipleToStore( + PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) +{ + /* Index the paths to copy by store path so the graph nodes below + can look up each path's info (NAR size, references) and source. */ + std::map> *> infosMap; + uint64_t bytesExpected = 0; + for (auto & item : pathsToCopy) { + bytesExpected += item.first.narSize; + infosMap.insert_or_assign(item.first.path, &item); + } + act.setExpected(actCopyPath, bytesExpected); + + std::atomic nrDone{0}; + std::atomic nrRunning{0}; + auto showProgress = [&, nrTotal = pathsToCopy.size()]() { act.progress(nrDone, nrTotal, nrRunning); }; + + /* The NarInfos produced by uploading the NARs, to be consumed when + writing the .narinfo files. Populated by the `UploadNar` nodes + and read by the corresponding `UploadNarInfo` nodes. */ + Sync>> narInfos_; + + /* The work graph has two kinds of nodes: uploading the NAR for a + path (which has no dependencies, since NARs are independent of + each other), and uploading the .narinfo for a path (which depends + on the corresponding NAR upload and on the .narinfo uploads of all + the path's references). Processing the latter in topological order + maintains the closure invariant: whenever a .narinfo exists, the + .narinfo files of all its references exist as well. */ + struct UploadNar + { + StorePath path; + uint64_t narSize; + + /* Order NAR uploads by descending size so that the largest + (and typically slowest) NARs are started first. */ + bool operator<(const UploadNar & other) const + { + return narSize != other.narSize ? narSize > other.narSize : path < other.path; + } + }; + + struct UploadNarInfo + { + StorePath path; + + bool operator<(const UploadNarInfo & other) const + { + return path < other.path; + } + }; + + /* `std::variant`'s `operator<` orders by alternative index first, so + all `UploadNar` nodes sort (and thus get enqueued) before any + `UploadNarInfo` node. + TODO: uploading the debug info and NAR listings could be turned into separate graph nodes as well. + */ + using Node = std::variant; + + std::set nodes; + for (auto & [path, item] : infosMap) { + nodes.insert(UploadNar{path, item->first.narSize}); + nodes.insert(UploadNarInfo{path}); + } + + processGraph( + nodes, + + [&](const Node & node) -> std::set { + return std::visit( + overloaded{ + [&](const UploadNar &) -> std::set { + /* NAR uploads have no dependencies. */ + return {}; + }, + [&](const UploadNarInfo & n) -> std::set { + std::set edges; + auto & info = infosMap.at(n.path)->first; + /* Wait for our own NAR to be uploaded ... */ + edges.insert(UploadNar{n.path, info.narSize}); + /* ... and for the .narinfo files of all + references that are part of this copy (other + references are already valid in the store). */ + for (auto & ref : info.references) { + if (ref != n.path && infosMap.count(ref)) + edges.insert(UploadNarInfo{ref}); + } + return edges; + }, + }, + node); + }, + + [&](const Node & node) { + checkInterrupt(); + std::visit( + overloaded{ + [&](const UploadNar & n) { + auto & [info, source_] = *infosMap.at(n.path); + + /* Make sure the Source object is destroyed when + we're done, e.g. to release the connection + lock held by LegacySSHStore::narFromPath(). */ + auto source = std::move(source_); + + if (repair || !isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + auto narInfo = uploadData(*source, repair, [&](HashResult nar) { + auto info2 = info; + info2.ultimate = false; + return info2; + }); + narInfos_.lock()->insert_or_assign(info.path, narInfo); + } + + nrDone++; + showProgress(); + }, + [&](const UploadNarInfo & n) { + auto & info = infosMap.at(n.path)->first; + if (!repair && isValidPath(info.path)) + return; + auto narInfo = narInfos_.lock()->at(n.path); + uploadNarInfo(narInfo); + }, + }, + node); + }); +} + StorePath BinaryCacheStore::addToStoreFromDump( Source & dump, std::string_view name, @@ -315,7 +481,8 @@ StorePath BinaryCacheStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { std::optional caHash; std::string nar; @@ -378,6 +545,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( }), nar.hash); info.narSize = nar.numBytesDigested; + info.provenance = provenance; return info; }) ->path; @@ -419,7 +587,13 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) stats.narReadBytes += narSize; }}; - auto decompressor = makeDecompressionSink(info->compression, uncompressedSink); + /* makeDecompressionSink used to treat empty strings as "none". It seems + impossible that it would actually end up here with an empty string though + (since an empty `Compression: ' is treated as bzip2 when parsed from a + .narinfo file and the narinfo disk cache wouldn't handle empty strings). + TODO: Revisit this and convert to an assert probably or even made + compression a non-optional field. */ + auto decompressor = makeDecompressionSink(info->compression.value_or(CompressionAlgo::none), uncompressedSink); try { getFile(info->url, *decompressor); @@ -507,6 +681,7 @@ StorePath BinaryCacheStore::addToStore( }), nar.hash); info.narSize = nar.numBytesDigested; + info.provenance = path.getProvenance(); return info; }) ->path; @@ -514,7 +689,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,7 +710,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -551,7 +729,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index f01911bcd002..dbbfa4af7995 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,9 +1,13 @@ #include "nix/store/build-result.hh" #include "nix/util/json-utils.hh" +#include "nix/util/provenance.hh" + #include namespace nix { +void BuildError::anchor() {} + void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) { // Allow selecting a subset of enum values @@ -20,7 +24,7 @@ void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) checkMismatch = true; break; case BuildResult::Failure::PermanentFailure: - // Also considered a permenant failure, it seems + // Also considered a permanent failure, it seems case BuildResult::Failure::InputRejected: permanentFailure = true; break; @@ -90,7 +94,7 @@ static BuildResult::Success::Status successStatusFromString(std::string_view str throw Error("unknown built result success status '%s'", str); } -static constexpr std::array, 12> failureStatusStrings{{ +static constexpr std::array, 13> failureStatusStrings{{ #define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} ENUM_ENTRY(PermanentFailure), ENUM_ENTRY(InputRejected), @@ -104,6 +108,7 @@ static constexpr std::array::to_json(json & res, const BuildResult & br) res["success"] = true; res["status"] = successStatusToString(success.status); res["builtOutputs"] = success.builtOutputs; + if (success.provenance) + res["provenance"] = success.provenance->to_json(); }, [&](const BuildResult::Failure & failure) { res["success"] = false; res["status"] = failureStatusToString(failure.status); res["errorMsg"] = failure.message(); res["isNonDeterministic"] = failure.isNonDeterministic; + if (failure.provenance) + res["provenance"] = failure.provenance->to_json(); }, }, br.inner); @@ -201,17 +210,24 @@ BuildResult adl_serializer::from_json(const json & _json) bool success = getBoolean(valueAt(json, "success")); std::string statusStr = getString(valueAt(json, "status")); + auto provenanceFromJson = [](const nlohmann::json * j) -> std::shared_ptr { + if (j && !j->is_null()) + return Provenance::from_json(*j); + return nullptr; + }; + if (success) { BuildResult::Success s; s.status = successStatusFromString(statusStr); s.builtOutputs = valueAt(json, "builtOutputs"); + s.provenance = provenanceFromJson(optionalValueAt(json, "provenance")); br.inner = std::move(s); } else { - br.inner = BuildResult::Failure{{ - .status = failureStatusFromString(statusStr), - .msg = HintFmt(getString(valueAt(json, "errorMsg"))), - .isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")), - }}; + br.inner = BuildResult::Failure{ + {.status = failureStatusFromString(statusStr), + .msg = HintFmt(getString(valueAt(json, "errorMsg"))), + .isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")), + .provenance = provenanceFromJson(optionalValueAt(json, "provenance"))}}; } return br; diff --git a/src/libstore/build/build-log.cc b/src/libstore/build/build-log.cc index a8fb64fc68bf..85b8877d6590 100644 --- a/src/libstore/build/build-log.cc +++ b/src/libstore/build/build-log.cc @@ -2,7 +2,7 @@ namespace nix { -BuildLog::BuildLog(size_t maxTailLines, std::unique_ptr act) +BuildLog::BuildLog(size_t maxTailLines, ref act) : maxTailLines(maxTailLines) , act(std::move(act)) { diff --git a/src/libstore/build/derivation-builder.cc b/src/libstore/build/derivation-builder.cc index 39ac40175f71..73404a08dcb8 100644 --- a/src/libstore/build/derivation-builder.cc +++ b/src/libstore/build/derivation-builder.cc @@ -25,3 +25,13 @@ void adl_serializer::to_json(json & json, const ExternalBuilder } } // namespace nlohmann + +namespace nix { + +void BuilderFailureError::anchor() {} + +void DerivationBuilder::anchor() {} + +DerivationBuilderCallbacks::~DerivationBuilderCallbacks() {} + +} // namespace nix diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 60ca5492c168..19f1cbadcc55 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -14,10 +14,11 @@ #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts +#include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" +#include "nix/util/current-process.hh" #include -#include #include #include #include @@ -29,10 +30,10 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) : Goal(worker, gaveUpOnSubstitution(storeDerivation)) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); @@ -65,7 +66,76 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } -static void runPostBuildHook( +namespace { + +struct LogSink : Sink +{ + Activity & act; + std::string currentLine; + + LogSink(Activity & act) + : act(act) + { + } + + void operator()(std::string_view data) override + { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() + { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() + { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } +}; + +} // namespace + +struct PostBuildHookState +{ + const std::string hook; + Activity act; + std::unique_ptr sink; + std::unique_ptr out; + Pid pid; + + PostBuildHookState(Logger & logger, const std::string hook, const std::string drvPath) + : hook(hook) + , act(logger, + lvlTalkative, + actPostBuildHook, + fmt("running post-build-hook '%s'", hook), + Logger::Fields{drvPath}) + , out(std::make_unique()) + { + out->create(); + sink = std::make_unique(act); + } + + void complete() + { + if (int ret = pid.wait()) { + throw Error("program \"%s\" %s", hook, statusToString(ret)); + } + } +}; + +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, @@ -99,7 +169,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) "dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i))); + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, true))); } co_await await(std::move(waitees)); @@ -143,7 +213,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) auto outMap = [&] { for (auto * drvStore : {&worker.evalStore, &worker.store}) if (drvStore->isValidPath(depDrvPath)) - return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + return deepQueryDerivationOutputMap(worker.store, depDrvPath, drvStore); assert(false); }(); @@ -304,9 +374,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -348,8 +417,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) WrongLocalStore wrongStore; - if (drv->platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv->platform) - && !drv->isBuiltin()) + if (drv->platform != settings.thisSystem.get() && drv->platform != "wasm32-wasip1" + && !settings.extraPlatforms.get().count(drv->platform) && !drv->isBuiltin()) wrongStore.badPlatform = WrongLocalStore::Pair{drv->platform, settings.thisSystem.get()}; { @@ -634,7 +703,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook( std::unique_ptr buildLog = std::make_unique( worker.settings.logLines, - std::make_unique( + make_ref( *logger, lvlInfo, actBuild, @@ -699,9 +768,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -762,7 +831,21 @@ Goal::Co DerivationBuildingGoal::buildWithHook( StorePathSet outputPaths; for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -787,6 +870,13 @@ Goal::Co DerivationBuildingGoal::buildLocally( #ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows throw UnimplementedError("building derivations is not yet implemented on Windows"); #else + auto msg = + fmt(buildMode == bmRepair ? "repairing outputs of '%s'" + : buildMode == bmCheck ? "checking outputs of '%s'" + : "building '%s'", + worker.store.printStorePath(drvPath)); + auto act = make_ref( + *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}); std::unique_ptr buildLog; std::unique_ptr logFile; @@ -797,15 +887,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( auto closeLogFile = [&]() { logFile.reset(); }; auto started = [&]() { - auto msg = - fmt(buildMode == bmRepair ? "repairing outputs of '%s'" - : buildMode == bmCheck ? "checking outputs of '%s'" - : "building '%s'", - worker.store.printStorePath(drvPath)); - buildLog = std::make_unique( - worker.settings.logLines, - std::make_unique( - *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1})); + buildLog = std::make_unique(worker.settings.logLines, act); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); }; @@ -814,6 +896,11 @@ Goal::Co DerivationBuildingGoal::buildLocally( DerivationBuilderUnique builder; Descriptor builderOut; + /* Get the provenance of the derivation, if available. */ + std::shared_ptr provenance; + if (auto info = worker.evalStore.maybeQueryPathInfo(drvPath)) + provenance = info->provenance; + // Will continue here while waiting for a build user below while (true) { @@ -891,6 +978,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( DerivationBuilderParams params{ .drvPath = drvPath, + .drvProvenance = provenance, .buildResult = buildResult, .drv = *drv, .drvOptions = drvOptions, @@ -900,6 +988,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( .defaultPathsInChroot = std::move(defaultPathsInChroot), .systemFeatures = worker.store.config.systemFeatures.get(), .desugaredEnv = std::move(desugaredEnv), + .act = act, }; /* If we have to wait and retry (see below), then `builder` will @@ -907,12 +996,12 @@ Goal::Co DerivationBuildingGoal::buildLocally( builder = localBuildCap.externalBuilder ? makeExternalDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params), *localBuildCap.externalBuilder) : makeDerivationBuilder( localBuildCap.localStore, - std::make_unique(*this, openLogFile, closeLogFile), + std::make_shared(*this, openLogFile, closeLogFile), std::move(params)); } @@ -956,9 +1045,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -992,7 +1081,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( worker.markContentsGood(output.outPath); outputPaths.insert(output.outPath); } - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -1000,29 +1103,26 @@ Goal::Co DerivationBuildingGoal::buildLocally( (unlinked) lock files. */ outputLocks.setDeletion(true); outputLocks.unlock(); - co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs)); + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs), provenance); } #endif } -static void runPostBuildHook( +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) { - auto hook = workerSettings.postBuildHook; - if (hook == "") - return; +#ifdef _WIN32 + throw UnimplementedError("post-build-hook is not implemented on Windows"); +#else + auto state = + std::make_unique(logger, workerSettings.postBuildHook.get(), store.printStorePath(drvPath)); + + auto hook = workerSettings.postBuildHook.get(); - Activity act( - logger, - lvlTalkative, - actPostBuildHook, - fmt("running post-build-hook '%s'", workerSettings.postBuildHook), - Logger::Fields{store.printStorePath(drvPath)}); - PushActivity pact(act.id); OsStringMap hookEnvironment = getEnvOs(); hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath))); @@ -1030,50 +1130,34 @@ static void runPostBuildHook( OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))))); hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); - struct LogSink : Sink - { - Activity & act; - std::string currentLine; + ProcessOptions processOptions; + processOptions.allowVfork = false; - LogSink(Activity & act) - : act(act) - { - } + state->pid = startProcess( + [&] { + replaceEnv(hookEnvironment); + if (dup2(state->out->writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); - void operator()(std::string_view data) override - { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } + Strings args_; + args_.push_front(hook); - void flushLine() - { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } + unix::closeExtraFDs(); - ~LogSink() - { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; + restoreProcessContext(); + + execvp(hook.c_str(), stringsToCharPtrs(args_).data()); - LogSink sink(act); + throw SysError("executing %s", PathFmt(hook)); + }, + processOptions); - runProgram2({ - .program = workerSettings.postBuildHook.get(), - .environment = hookEnvironment, - .standardOut = &sink, - .mergeStderrToStdout = true, - }); + state->out->writeSide.close(); + + return state; +#endif } BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog) @@ -1094,7 +1178,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur msg += line; msg += "\n"; } - auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) ? "nix log" : "nix-store -l"; + auto nixLogCommand = "nix log"; // The command is on a separate line for easy copying, such as with triple click. // This message will be indented elsewhere, so removing the indentation before the // command will not put it at the start of the line unfortunately. @@ -1186,12 +1270,7 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings auto baseName = std::string(baseNameOf(store.printStorePath(drvPath))); - std::filesystem::path logDir; - if (auto localStore = dynamic_cast(&store)) - logDir = localStore->config->logDir.get(); - else - logDir = logSettings.nixLogDir; - auto dir = logDir / LocalFSStore::drvsLogDir / baseName.substr(0, 2); + auto dir = store.config.getLogDir() / LocalFSStore::drvsLogDir / baseName.substr(0, 2); createDirs(dir); auto logFileName = dir / (baseName.substr(2) + (logSettings.compressLog ? ".bz2" : "")); @@ -1276,12 +1355,12 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, - .status = PathStatus::Valid, + .status = worker.store.isValidPath(real->outPath) ? PathStatus::Valid : PathStatus::Absent, }; } else if (info.known && info.known->isValid()) { // We know the output because it's a static output of the @@ -1319,7 +1398,8 @@ DerivationBuildingGoal::checkPathValidity(std::map & return {allValid, validOutputs}; } -Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs) +Goal::Done DerivationBuildingGoal::doneSuccess( + BuildResult::Success::Status status, SingleDrvOutputs builtOutputs, std::shared_ptr provenance) { mcRunningBuilds.reset(); @@ -1328,11 +1408,21 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat worker.updateProgress(); - return Goal::doneSuccess( + auto res = Goal::doneSuccess( BuildResult::Success{ .status = status, .builtOutputs = std::move(builtOutputs), + .provenance = provenance, }); + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + + return res; } Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) @@ -1345,7 +1435,16 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) worker.updateProgress(); - return Goal::doneFailure(ecFailed, std::move(ex)); + auto res = Goal::doneFailure(ecFailed, std::move(ex)); + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + + return res; } } // namespace nix diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index c422897e624e..8421642042b3 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -7,12 +7,55 @@ namespace nix { +void checkCAFixedOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + Activity & act) +{ + if (const auto * dof = std::get_if(&outputSpec.raw)) { + auto & wanted = dof->ca.hash; + + /* Check wanted hash */ + assert(info.ca); + auto & got = info.ca->hash; + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + act.result( + resHashMismatch, + { + {"storePath", store.printStorePath(drvPath)}, + {"wanted", wanted}, + {"got", got}, + }); + throw BuildError( + BuildResult::Failure::HashMismatch, + "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true)); + } + if (!info.references.empty()) { + auto numViolations = info.references.size(); + throw BuildError( + BuildResult::Failure::HashMismatch, + "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*info.references.begin())); + } + } +} + void checkOutputs( Store & store, const StorePath & drvPath, const decltype(Derivation::outputs) & drvOutputs, const decltype(DerivationOptions::outputChecks) & outputChecks, - const std::map & outputs) + const std::map & outputs, + Activity & act) { std::map outputsByPath; for (auto & output : outputs) @@ -27,32 +70,7 @@ void checkOutputs( auto * outputSpec = get(drvOutputs, outputName); assert(outputSpec); - if (const auto * dof = std::get_if(&outputSpec->raw)) { - auto & wanted = dof->ca.hash; - - /* Check wanted hash */ - assert(info.ca); - auto & got = info.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - throw BuildError( - BuildResult::Failure::HashMismatch, - "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true)); - } - if (!info.references.empty()) { - auto numViolations = info.references.size(); - throw BuildError( - BuildResult::Failure::HashMismatch, - "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*info.references.begin())); - } - } + checkCAFixedOutput(store, drvPath, *outputSpec, info, act); /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely @@ -186,7 +204,8 @@ void checkOutputs( std::visit( overloaded{ [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, - [&](const std::map::OutputChecks> & checksPerOutput) { + [&](const std::map::OutputChecks, std::less<>> & + checksPerOutput) { if (auto outputChecks = get(checksPerOutput, outputName)) applyChecks(*outputChecks); diff --git a/src/libstore/build/derivation-check.hh b/src/libstore/build/derivation-check.hh index 01e6c5d56383..1832264ea1a9 100644 --- a/src/libstore/build/derivation-check.hh +++ b/src/libstore/build/derivation-check.hh @@ -7,6 +7,18 @@ namespace nix { +/** + * If outputSpec is a CAFixed output, check that the actual output described in + * info meets the requirements for a CAFixed output. Do nothing if outputSpec is + * not a CAFixed output. + */ +void checkCAFixedOutput( + StoreDirConfig & store, + const StorePath & drvPath, + const DerivationOutput & outputSpec, + const ValidPathInfo & info, + Activity & act); + /** * Check that outputs meets the requirements specified by the * 'outputChecks' attribute (or the legacy @@ -22,6 +34,7 @@ void checkOutputs( const StorePath & drvPath, const decltype(Derivation::outputs) & drvOutputs, const decltype(DerivationOptions::outputChecks) & drvOptions, - const std::map & outputs); + const std::map & outputs, + Activity & act); } // namespace nix diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4f99928d7850..55bf3923903b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2,33 +2,23 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-resolution-goal.hh" -#ifndef _WIN32 // TODO enable build hook on Windows -# include "nix/store/build/hook-instance.hh" -# include "nix/store/build/derivation-builder.hh" -#endif -#include "nix/util/processes.hh" -#include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" -#include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" // Don't remove is actually needed -#include "nix/store/globals.hh" +#include "nix/store/outputs-query.hh" -#include #include #include #include #include -#include "nix/util/strings.hh" - namespace nix { DerivationGoal::DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -36,13 +26,7 @@ DerivationGoal::DerivationGoal( : Goal(worker, haveDerivation(storeDerivation)) , drvPath(drvPath) , wantedOutput(wantedOutput) - , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} + , drv{std::move(drv)} , buildMode(buildMode) { @@ -78,9 +62,12 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); + StorePathSet outputPaths; for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) - worker.store.addTempRoot(*i.second.second); + outputPaths.insert(*i.second.second); + + worker.store.addTempRoots(outputPaths); /* We don't yet have any safe way to cache an impure derivation at this step. */ @@ -102,12 +89,14 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { if (!checkResult) { - DrvOutput id{outputHash, wantedOutput}; + DrvOutput id{drvPath, wantedOutput}; auto g = worker.makeDrvOutputSubstitutionGoal(id); waitees.insert(g); co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -122,11 +111,14 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto * cap = getDerivationCA(*drv); waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( checkResult->first.outPath, + false, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional{*cap} : std::nullopt))); } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); @@ -160,20 +152,27 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); + /* We'll handle the error below. */ + resolutionGoal->preserveFailure = true; + co_await await({resolutionGoal}); + if (nrFailed != 0) { - co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); + auto * failure = resolutionGoal->buildResult.tryGetFailure(); + assert(failure); + co_return doneFailure(*failure); } if (resolutionGoal->resolvedDrv) { auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv; - auto resolvedDrvGoal = - worker.makeDerivationGoal(pathResolved, drvResolved, wantedOutput, buildMode, /*storeDerivation=*/true); + auto resolvedDrvGoal = worker.makeDerivationGoal( + pathResolved, + make_ref(drvResolved), + wantedOutput, + buildMode, + /*storeDerivation=*/true); + { Goals waitees{resolvedDrvGoal}; co_await await(std::move(waitees)); @@ -186,12 +185,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -200,7 +194,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -208,7 +202,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -220,18 +214,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) wantedOutput); }(); - if (!drv->type().isImpure()) { - Realisation newRealisation{ - realisation, - { - .drvHash = *outputHash, - .outputName = wantedOutput, - }}; - newRealisation.signatures.clear(); - worker.store.signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - } - auto status = success.status; if (status == BuildResult::Success::AlreadyValid) status = BuildResult::Success::ResolvesToAlreadyValid; @@ -247,18 +229,17 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation); + auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); @@ -270,16 +251,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -298,16 +270,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (success.builtOutputs.count(wantedOutput) == 0) { debug( "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput); - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } } } @@ -328,7 +291,7 @@ Goal::Co DerivationGoal::repairClosure() auto outputs = [&] { for (auto * drvStore : {&worker.evalStore, &worker.store}) if (drvStore->isValidPath(drvPath)) - return worker.store.queryDerivationOutputMap(drvPath, drvStore); + return deepQueryDerivationOutputMap(worker.store, drvPath, drvStore); OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) @@ -376,7 +339,7 @@ Goal::Co DerivationGoal::repairClosure() worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, false, Repair))); else waitees.insert(worker.makeGoal( DerivedPath::Built{ @@ -404,7 +367,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -444,7 +407,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -472,20 +435,20 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke worker.updateProgress(); - return Goal::doneSuccess( + auto res = Goal::doneSuccess( BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }); + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + + return res; } Goal::Done DerivationGoal::doneFailure(BuildError ex) @@ -498,6 +461,12 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex) worker.updateProgress(); + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + {ex}, DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + return Goal::doneFailure(ecFailed, std::move(ex)); } diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e18563..6f55ea29d96e 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); @@ -22,60 +22,49 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; - - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; + + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 58a43c043a2a..14864ed052f8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -2,6 +2,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/derivations.hh" +#include +#include + namespace nix { DerivationTrampolineGoal::DerivationTrampolineGoal( @@ -46,18 +49,17 @@ void DerivationTrampolineGoal::commonInit() DerivationTrampolineGoal::~DerivationTrampolineGoal() {} -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, - [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, - }, - req.raw()); -} - std::string DerivationTrampolineGoal::key() { + auto pathPartOfReq = [](this const auto & self, const SingleDerivedPath & req) -> StorePath { + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return self(*bfd.drvPath); }, + }, + req.raw()); + }; + return "da$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{ .drvPath = drvReq, .outputs = wantedOutputs, @@ -145,31 +147,87 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }, wantedOutputs.raw); + /* Must have at least one wanted output. This is assumed below. */ + assert(!resolvedWantedOutputs.empty()); + Goals concreteDrvGoals; /* Build this step! */ + auto sharedDrv = make_ref(std::move(drv)); + for (auto & output : resolvedWantedOutputs) { - auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false)); + auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); g->preserveFailure = true; /* We will finish with it ourselves, as if we were the derivational goal. */ concreteDrvGoals.insert(std::move(g)); } - // Copy on purpose - co_await await(Goals(concreteDrvGoals)); + co_await await(concreteDrvGoals); trace("outer build done"); - auto & g = *concreteDrvGoals.begin(); - buildResult = g->buildResult; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & g2 : concreteDrvGoals) - if (auto * successP2 = g2->buildResult.tryGetSuccess()) - for (auto && [x, y] : successP2->builtOutputs) - successP->builtOutputs.insert_or_assign(x, y); + if (nrFailed != 0) { + auto gi = std::ranges::find_if(concreteDrvGoals, [](const GoalPtr & goal) -> bool { + auto exitCode = goal->exitCode; + /* Note that without --keep-going waitees might be cancelled before + we are woken up. */ + return exitCode != ecBusy && exitCode != ecSuccess; + }); + + const Goal * g = gi->get(); + assert(gi != concreteDrvGoals.end() && "expected a failing goal"); + auto exitCode = g->exitCode; + const auto * failure = g->buildResult.tryGetFailure(); + assert(failure && "failing goal does not report a failed build result"); + + /* Report the exit status of *some* failing goal. This might not be strictly + correct, since multiple subgoals can fail independently, but this should be + a good enough heuristic without --keep-going. */ + co_return doneFailure(exitCode, *failure); + } - co_return amDone(g->exitCode); + SingleDrvOutputs outputs; + + auto successes = std::views::transform(concreteDrvGoals, [](const GoalPtr & a) -> const BuildResult::Success & { + auto * success = a->buildResult.tryGetSuccess(); + assert(success && "goal succeeded, but some waitees do not report a successful status"); + return *success; + }); + + for (const auto & success : successes) + std::ranges::copy(success.builtOutputs, std::inserter(outputs, outputs.end())); + + auto statuses = successes | std::views::transform(&BuildResult::Success::status); + + /* Aggregate the status code. If some outputs we already valid, but we had + to build/substitute the other ones, report it as the smallest common + denominator. */ + auto compareSuccesses = [](auto a, auto b) { + /* This is technically an identity mapping of the underlying values, but + it would be worse to rely on the enum ordering here. */ + auto toPriority = [](auto st) { + using enum BuildResult::Success::Status; + switch (st) { + case Built: + return 0; + case Substituted: + return 1; + case AlreadyValid: + return 2; + case ResolvesToAlreadyValid: + return 3; + default: + unreachable(); + } + }; + return toPriority(a) < toPriority(b); + }; + + co_return doneSuccess({ + .status = std::ranges::min(statuses, compareSuccesses), + .builtOutputs = std::move(outputs), + }); } } // namespace nix diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 71f0ae1e7db4..ed653fed7555 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,7 +1,7 @@ +#include "goal-impl.hh" + #include "nix/store/build/drv-output-substitution-goal.hh" -#include "nix/util/finally.hh" #include "nix/store/build/worker.hh" -#include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" namespace nix { @@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -30,55 +30,9 @@ Goal::Co DrvOutputSubstitutionGoal::init() for (const auto & sub : subs) { trace("trying next substituter"); - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - auto outPipe = std::make_shared(); -#ifndef _WIN32 - outPipe->create(); -#else - outPipe->createAsyncPipe(worker.ioport.get()); -#endif - - auto promise = std::make_shared>>(); - - sub->queryRealisation( - id, {[outPipe(outPipe), promise(promise)](std::future> res) { - try { - Finally updateStats([&]() { outPipe->writeSide.close(); }); - promise->set_value(res.get()); - } catch (...) { - promise->set_exception(std::current_exception()); - } - }}); - - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe->readSide.get() -#else - &*outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); - } - } - - worker.childTerminated(this); - try { - outputInfo = promise->get_future().get(); + outputInfo = co_await AsyncCallback>( + [sub, id = this->id](auto cb) { sub->queryRealisation(id, std::move(cb)); }); } catch (std::exception & e) { printError(e.what()); substituterFailed = true; @@ -93,7 +47,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -108,7 +63,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } } // namespace nix diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e6d1f62d786a..dbc688cb033b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,7 +2,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" -#include "nix/store/local-store.hh" #include "nix/util/strings.hh" namespace nix { @@ -65,11 +64,6 @@ std::vector Store::buildPathsWithResults( results.reserve(state.size()); for (auto & [req, goalPtr] : state) { - /* Goals that were never started or were cancelled have exitCode - ecBusy and a default buildResult with empty errorMsg. Skip them - to avoid reporting spurious failures with empty messages. */ - if (goalPtr->exitCode == Goal::ecBusy) - continue; results.emplace_back( KeyedBuildResult{ goalPtr->buildResult, @@ -104,7 +98,7 @@ void Store::ensurePath(const StorePath & path) return; Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path); + GoalPtr goal = worker.makePathSubstitutionGoal(path, true); Goals goals = {goal}; worker.run(goals); @@ -119,7 +113,7 @@ void Store::ensurePath(const StorePath & path) void Store::repairPath(const StorePath & path) { Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); + GoalPtr goal = worker.makePathSubstitutionGoal(path, true, Repair); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/build/goal-impl.hh b/src/libstore/build/goal-impl.hh new file mode 100644 index 000000000000..b35d2b5df632 --- /dev/null +++ b/src/libstore/build/goal-impl.hh @@ -0,0 +1,52 @@ +#pragma once +#include "nix/store/build/goal.hh" +#include "nix/store/build/worker.hh" + +namespace nix { + +template +auto Goal::promise_type::await_transform(AsyncCallback && ac) +{ + struct Awaiter + { + fun)> fn; + std::shared_ptr> promise; + + bool await_ready() + { + return false; + } + + void await_suspend(handle_type h) + { + auto goal = h.promise().goal; + promise = std::make_shared>(); + + fn(Callback([promise = promise, + maybeWaker = goal->worker.getCrossThreadWaker(), + goalWeak = goal->weak_from_this()](std::future res) { + try { + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + + /* The Worker might have already died (and the waker with it) by + the time the callback fired. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(goalWeak); + })); + + goal->worker.waitForCompletion(goal->shared_from_this()); + } + + T await_resume() + { + return promise->get_future().get(); + } + }; + + return Awaiter{.fn = std::move(ac.fn)}; +} + +} // namespace nix diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ef3501b00916..5b1be405d246 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -4,24 +4,29 @@ namespace nix { +void WorkerSettings::anchor() {} + TimedOut::TimedOut(time_t maxDuration) : CloneableError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration) , maxDuration(maxDuration) { } +void TimedOut::anchor() {} + +void Goal::anchor() {} + using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +34,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,23 +57,28 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() @@ -157,9 +167,7 @@ std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) bool CompareGoalPtrs::operator()(const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; + return a->keyCached() < b->keyCached(); } void addToWeakGoals(WeakGoals & goals, GoalPtr p) @@ -275,22 +283,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } @@ -308,6 +313,13 @@ Goal::Co Goal::waitForAWhile() co_return Return{}; } +Goal::Co Goal::waitUntilWoken() +{ + worker.waitForCompletion(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + Goal::Co Goal::waitForBuildSlot() { worker.waitForBuildSlot(shared_from_this()); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d58bec8851d5..de49582cd4c9 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,18 +1,25 @@ +#include "goal-impl.hh" + #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" -#include "nix/util/finally.hh" +#include "nix/store/worker-settings.hh" #include "nix/util/signals.hh" -#include "nix/store/globals.hh" +#include "nix/util/callback.hh" -#include +#include namespace nix { PathSubstitutionGoal::PathSubstitutionGoal( - const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) + const StorePath & storePath, + Worker & worker, + bool pathRequired, + RepairFlag repair, + std::optional ca) : Goal(worker, init()) , storePath(storePath) + , pathRequired(pathRequired) , repair(repair) , ca(ca) { @@ -26,6 +33,18 @@ PathSubstitutionGoal::~PathSubstitutionGoal() cleanup(); } +Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure failure) +{ + auto res = Goal::doneFailure(result, std::move(failure)); + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); + + return res; +} + Goal::Co PathSubstitutionGoal::init() { trace("init"); @@ -72,9 +91,9 @@ Goal::Co PathSubstitutionGoal::init() } try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath & e) { + info = co_await AsyncCallback>( + [sub, path = subPath.value_or(storePath)](auto cb) { sub->queryPathInfo(path, std::move(cb)); }); + } catch (InvalidPath &) { continue; } catch (SubstituterDisabled & e) { continue; @@ -127,7 +146,7 @@ Goal::Co PathSubstitutionGoal::init() paths referenced by this one. */ for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - waitees.insert(worker.makePathSubstitutionGoal(i)); + waitees.insert(worker.makePathSubstitutionGoal(i, pathRequired)); co_await await(std::move(waitees)); @@ -146,7 +165,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } @@ -155,7 +174,7 @@ Goal::Co PathSubstitutionGoal::init() In that case the calling derivation should just do a build. */ co_return doneFailure( - substituterFailed ? ecFailed : ecNoSubstituters, + substituterFailed || pathRequired ? ecFailed : ecNoSubstituters, BuildResult::Failure{{ .status = BuildResult::Failure::NoSubstituters, .msg = HintFmt( @@ -204,65 +223,65 @@ Goal::Co PathSubstitutionGoal::tryToRun( auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); -#ifndef _WIN32 - outPipe.create(); -#else - outPipe.createAsyncPipe(worker.ioport.get()); -#endif - - auto promise = std::promise(); - - thr = std::thread([this, &promise, &subPath, &sub]() { + auto promise = std::promise>(); + auto future = promise.get_future(); + + /* Be careful with ownership. cleanup() doesn't signal the worker thread + to cleanly shutdown, so the worker can die while the thread is still + running. That's why we use weak_ptr for everything that is owned by the + Worker. */ + thr = std::thread([weakGoal = weak_from_this(), + promise = std::move(promise), + subPath, + storePath = storePath, + repair = repair, + sub, + maybeWaker = worker.getCrossThreadWaker(), + maybeWorkerStore = worker.store.weak_from_this()]() mutable { try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); + /* The Worker might have died while we were starting up. */ + auto workerStore = maybeWorkerStore.lock(); + if (!workerStore) + return; Activity act( *logger, actSubstitute, - Logger::Fields{worker.store.printStorePath(storePath), sub->config.getHumanReadableURI()}); + Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}); PushActivity pact(act.id); - copyStorePath(*sub, worker.store, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); + promise.set_value( + copyStorePath(*sub, *workerStore, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs)); } catch (...) { promise.set_exception(std::current_exception()); } + + /* The Worker might have already died (and the waker with it) by the + time we finished. N.B. if enqueueing to the waker throws, we better + std::terminate, since something has gone very wrong. This intentionally + lets the thread crash on exceptions for that reason. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(weakGoal); }); - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe.readSide.get() -#else - &outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Substitution doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); // Substitution doesn't use timeouts - } - } + /* Use up the substitution slot. */ + worker.childStarted(shared_from_this(), /*channels=*/{}, /*inBuildSlot=*/true, /*respectTimeouts=*/false); + /* Suspend until the thread finishes. */ + co_await waitUntilWoken(); trace("substitute finished"); thr.join(); worker.childTerminated(this); + std::shared_ptr provenance; + try { - promise.get_future().get(); + auto info = future.get(); + if (info) + provenance = info->provenance; } catch (std::exception & e) { /* Cause the parent build to fail unless --fallback is given, or the substitute has disappeared. The latter case behaves @@ -303,7 +322,12 @@ Goal::Co PathSubstitutionGoal::tryToRun( worker.updateProgress(); - co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::Substituted}); + auto success = BuildResult::Success{.status = BuildResult::Success::Substituted, .provenance = provenance}; + + logger->result( + getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult({success}, DerivedPath::Opaque{storePath}))); + + co_return doneSuccess(std::move(success)); } void PathSubstitutionGoal::cleanup() @@ -314,8 +338,6 @@ void PathSubstitutionGoal::cleanup() thr.join(); worker.childTerminated(this, JobCategory::Substitution); } - - outPipe.close(); } catch (...) { ignoreExceptionInDestructor(); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ffb1027b42c5..fbf9b8e62f39 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -17,7 +17,9 @@ namespace nix { Worker::Worker(Store & store, Store & evalStore) - : act(*logger, actRealise) + /* Can't use make_ref, because the constructor is private. */ + : wakerState(ref(new Waker{})) + , act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) #ifdef _WIN32 @@ -33,6 +35,7 @@ Worker::Worker(Store & store, Store & evalStore) #ifdef _WIN32 if (!ioport) throw windows::WinError("CreateIoCompletionPort"); + wakerState->ioport = ioport.get(); #endif nrLocalBuilds = 0; nrSubstitutions = 0; @@ -58,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; @@ -85,31 +88,38 @@ std::shared_ptr Worker::makeDerivationTrampolineGoal( std::shared_ptr Worker::makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation) { return initGoalIfNeeded( - derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode, storeDerivation); + derivationGoals[drvPath][wantedOutput], + drvPath, + std::move(drv), + wantedOutput, + *this, + buildMode, + storeDerivation); } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } std::shared_ptr Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) { - return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode, storeDerivation); + return initGoalIfNeeded( + derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); } -std::shared_ptr -Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal( + const StorePath & path, bool pathRequired, RepairFlag repair, std::optional ca) { - return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca); + return initGoalIfNeeded(substitutionGoals[path], path, *this, pathRequired, repair, ca); } std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput & id) @@ -125,70 +135,37 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) return makeDerivationTrampolineGoal(bfd.drvPath, bfd.outputs, buildMode); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { - return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + return makePathSubstitutionGoal(bo.path, false, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); } -/** - * This function is polymorphic (both via type parameters and - * overloading) and recursive in order to work on a various types of - * trees - * - * @return Whether the tree node we are processing is not empty / should - * be kept alive. In the case of this overloading the node in question - * is the leaf, the weak reference itself. If the weak reference points - * to the goal we are looking for, our caller can delete it. In the - * inductive case where the node is an interior node, we'll likewise - * return whether the interior node is non-empty. If it is empty - * (because we just deleted its last child), then our caller can - * likewise delete it. - */ -template -static bool removeGoal(std::shared_ptr goal, std::weak_ptr & gp) -{ - return gp.lock() != goal; -} - -template -static bool removeGoal(std::shared_ptr goal, std::map & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); i != goalMap.end();) { - if (!removeGoal(goal, i->second)) - i = goalMap.erase(i); - else - ++i; - } - return !goalMap.empty(); -} - -template -static bool -removeGoal(std::shared_ptr goal, typename DerivedPathMap>>::ChildNode & node) -{ - bool valueKeep = removeGoal(goal, node.value); - bool childMapKeep = removeGoal(goal, node.childMap); - return valueKeep || childMapKeep; -} - void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationTrampolineGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvResolutionGoal, derivationResolutionGoals); - else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvBuildingGoal, derivationBuildingGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + derivationTrampolineGoals.removeSlot(*drvGoal->drvReq, [&](auto & node) { + node.value.erase(drvGoal->wantedOutputs); + /* Return true if ancestors don't need to be pruned. */ + return !node.value.empty(); + }); + } else if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + if (auto it = derivationGoals.find(drvGoal->drvPath); it != derivationGoals.end()) { + it->second.erase(drvGoal->wantedOutput); + if (it->second.empty()) + derivationGoals.erase(it); + } + } else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) { + derivationResolutionGoals.erase(drvResolutionGoal->drvPath); + } else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) { + derivationBuildingGoals.erase(drvBuildingGoal->drvPath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + substitutionGoals.erase(subGoal->storePath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + drvOutputSubstitutionGoals.erase(subGoal->id); + } else { + unreachable(); + } if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); @@ -197,15 +174,6 @@ void Worker::removeGoal(GoalPtr goal) if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) - wakeUp(goal); - } - - waitingForAnyGoal.clear(); } void Worker::wakeUp(GoalPtr goal) @@ -244,21 +212,21 @@ void Worker::childStarted( nrLocalBuilds++; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } } -void Worker::childTerminated(Goal * goal, bool wakeSleepers) +void Worker::childTerminated(Goal * goal) { - childTerminated(goal, goal->jobCategory(), wakeSleepers); + childTerminated(goal, goal->jobCategory()); } -void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers) +void Worker::childTerminated(Goal * goal, JobCategory jobCategory) { + // FIXME: Inefficient. Make children a map from Goal -> Child instead. auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; }); if (i == children.end()) return; @@ -274,75 +242,51 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee nrLocalBuilds--; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } children.erase(i); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) - wakeUp(goal); - } - - wantingToBuild.clear(); - } } void Worker::waitForBuildSlot(GoalPtr goal) { goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) - || (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ + + bool slotAvailable = [&] { + if (goal->jobCategory() == JobCategory::Substitution) + return getNrSubstitutions() < settings.maxSubstitutionJobs; + else + return getNrLocalBuilds() < settings.maxBuildJobs; + }(); + + if (slotAvailable) + wakeUp(goal); /* Can do it right away. */ else - addToWeakGoals(wantingToBuild, goal); + addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } -void Worker::waitForAnyGoal(GoalPtr goal) +void Worker::waitForAWhile(GoalPtr goal) { - debug("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); + goal->trace("wait for a while"); + addToWeakGoals(waitingForAWhile, goal); } -void Worker::waitForAWhile(GoalPtr goal) +void Worker::waitForCompletion(GoalPtr goal) { - debug("wait for a while"); - addToWeakGoals(waitingForAWhile, goal); + goal->trace("waiting for completion callback"); + addToWeakGoals(waitingForCompletion, goal); } void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? @@ -359,19 +303,50 @@ void Worker::run(const Goals & _topGoals) awake2.insert(goal); } awake.clear(); + for (auto & goal : awake2) { checkInterrupt(); + + std::chrono::time_point startTime; + if (verbosity >= lvlVomit) + startTime = std::chrono::steady_clock::now(); + goal->work(); + + /* Useful for tracing which goals hod the event loop. */ + vomit( + "worker event loop worked goal '%1%' for %2$.3fms", + goal->name, + std::chrono::duration_cast>( + std::chrono::steady_clock::now() - startTime) + .count()); + if (topGoals.empty()) break; // stuff may have been cancelled } + + auto wakeSlotWaiters = [this](WeakGoals & waiting, size_t running, size_t limit) { + auto it = waiting.begin(); + while (it != waiting.end() && running < limit) { + auto goal = it->lock(); + it = waiting.erase(it); + if (!goal) + continue; + wakeUp(goal); + ++running; + } + }; + + wakeSlotWaiters( + wantingToSubstitute, getNrSubstitutions(), std::max(1, settings.maxSubstitutionJobs)); + wakeSlotWaiters(wantingToBuild, getNrLocalBuilds(), settings.maxBuildJobs); } if (topGoals.empty()) break; - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) + /* Wait for input or completion callbacks. */ + if (!children.empty() || !waitingForAWhile.empty() || !waitingForCompletion.empty()) waitForInput(); else if (awake.empty() && 0U == settings.maxBuildJobs) { if (Machine::parseConfig({nix::settings.thisSystem}, nix::settings.getWorkerSettings().builders).empty()) @@ -393,6 +368,7 @@ void Worker::run(const Goals & _topGoals) --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || wantingToSubstitute.empty()); assert(!settings.keepGoing || children.empty()); } @@ -463,6 +439,17 @@ void Worker::waitForInput() state.fdToPollStatus[j] = state.pollStatus.size() - 1; } } + + { + auto wakeupPipeFd = wakerState->wakeupPipe.pipe.readSide.get(); + state.pollStatus.push_back( + pollfd{ + .fd = wakeupPipeFd, + .events = POLLIN, + }); + + state.fdToPollStatus[wakeupPipeFd] = state.pollStatus.size() - 1; + } #endif state.poll( @@ -508,6 +495,17 @@ void Worker::waitForInput() } } +#ifndef _WIN32 + std::set wakerChannels{wakerState->wakeupPipe.pipe.readSide.get()}; + state.iterate( + wakerChannels, + [&](Descriptor k, std::string_view data) { wakerState->wakeAll(*this); }, + [](Descriptor fd) { unreachable(); }); +#else + /* Slightly less optimal on windows. We don't use a wakeup pipe and signal the ioport directly. */ + wakerState->wakeAll(*this); +#endif + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { @@ -519,6 +517,39 @@ void Worker::waitForInput() } } +std::weak_ptr Worker::getCrossThreadWaker() +{ + return wakerState.get_ptr(); +} + +void Worker::Waker::wakeAll(Worker & worker) +{ + /* Wake up all goals that have been enqueued by asynchronous completion callbacks. */ + auto wakeupQueue(wakeupQueue_.lock()); +#ifndef _WIN32 + wakeupPipe.drain(); +#endif + while (!wakeupQueue->empty()) { + auto ptr = wakeupQueue->front().lock(); + wakeupQueue->pop(); + if (ptr) { + worker.waitingForCompletion.erase(ptr); + worker.wakeUp(ptr); + } + } +} + +void Worker::Waker::enqueue(WeakGoalPtr goal) +{ + wakeupQueue_.lock()->push(goal); +#ifdef _WIN32 + PostQueuedCompletionStatus( + ioport, /*dwNumberOfBytesTransferred=*/0, /*dwCompletionKey=*/0, /*lpOverlapped=*/nullptr); +#else + wakeupPipe.notify(); +#endif +} + bool Worker::pathContentsGood(const StorePath & path) { auto i = pathContentsGoodCache.find(path); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 3dd66be2ccae..2ad98fa92741 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -10,6 +10,8 @@ namespace nix { +void BuildEnvFileConflictError::anchor() {} + RegisterBuiltinBuilder::BuiltinBuilders & RegisterBuiltinBuilder::builtinBuilders() { static RegisterBuiltinBuilder::BuiltinBuilders builders; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1ff66eb89fa4..fd38b5a4afd8 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,7 +1,6 @@ #include "nix/store/builtins.hh" #include "nix/store/filetransfer.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/util/file-system.hh" @@ -14,12 +13,13 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ if (ctx.netrcData != "") { - fileTransferSettings.netrcFile = "netrc"; + fileTransferSettings.netrcFile = ctx.tmpDirInSandbox / "netrc"; writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); } - fileTransferSettings.caFile = "ca-certificates.crt"; - writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600); + auto caFilePath = ctx.tmpDirInSandbox / "ca-certificates.crt"; + fileTransferSettings.caFile = std::optional{caFilePath}; + writeFile(caFilePath, ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); if (!out) @@ -54,7 +54,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) } #endif - auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? CompressionAlgo::xz : CompressionAlgo::none, sink); fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897992..931d7ab7d959 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -2,40 +2,21 @@ -- Won't be loaded unless the experimental feature `ca-derivations` -- is enabled -create table if not exists Realisations ( +-- Why the `*V` tables +-- +-- We are trying to keep different versions of the experiment to have +-- completely independent extra schemas from one another. This will +-- enable people to switch between versions of the experiment (including +-- newer to older) without migrating between them, but at the cost +-- of having many abandoned tables lying around. Closer to the end of +-- the experiment, we'll provide guidance on how to clean this up. + +create table if not exists BuildTraceV3 ( id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" - outputPath integer not null, - signatures text, -- space-separated list - foreign key (outputPath) references ValidPaths(id) on delete cascade -); - -create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict + outputPath text not null, + signatures text -- space-separated list ); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); +create index if not exists IndexBuildTraceV3 on BuildTraceV3(drvPath, outputName); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index a0afe948d649..e9bd5fa56459 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,10 @@ #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" -#include "nix/store/store-api.hh" #include "nix/store/build-result.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/signature/local-keys.hh" #include @@ -27,13 +26,13 @@ void CommonProto::Serialise::write( StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { - return store.parseStorePath(readString(conn.from)); + return conn.shortStorePaths ? StorePath(readString(conn.from)) : store.parseStorePath(readString(conn.from)); } void CommonProto::Serialise::write( const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { - conn.to << store.printStorePath(storePath); + conn.to << (conn.shortStorePaths ? storePath.to_string() : store.printStorePath(storePath)); } ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) @@ -47,45 +46,19 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); - return s == "" ? std::optional{} : store.parseStorePath(s); + return s == "" ? std::optional{} : conn.shortStorePaths ? StorePath(s) : store.parseStorePath(s); } void CommonProto::Serialise>::write( const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); + conn.to + << (storePathOpt ? (conn.shortStorePaths ? storePathOpt->to_string() : store.printStorePath(*storePathOpt)) + : ""); } std::optional @@ -134,6 +107,8 @@ constexpr static BuildResultStatus buildResultStatusTable[] = { BuildResultFailureStatus::NotDeterministic, // 12 BuildResultSuccessStatus::ResolvesToAlreadyValid, // 13 BuildResultFailureStatus::NoSubstituters, // 14 + BuildResultFailureStatus::HashMismatch, // 15 + BuildResultFailureStatus::Cancelled, // 16 }; BuildResultStatus diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index fd61b12efeee..db3677151416 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -4,11 +4,13 @@ namespace nix { CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , authority(authority) { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0e6668bc0b23..24cb9207113a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,7 +3,6 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -16,9 +15,13 @@ #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" +#include "nix/store/active-builds.hh" +#include "nix/util/provenance.hh" +#include "nix/util/async.hh" + +#include #ifndef _WIN32 // TODO need graceful async exit support on Windows? # include "nix/util/monitor-fd.hh" @@ -43,6 +46,8 @@ Sink & operator<<(Sink & sink, const Logger::Fields & fields) return sink; } +namespace { + /* Logger that forwards log messages to the client, *if* we're in a state where the protocol allows it (i.e., when canSendStderr is true). */ @@ -66,7 +71,7 @@ struct TunnelLogger : public Logger { } - void enqueueMsg(const std::string & s) + void enqueueMsg(const std::string & s) noexcept { auto state(state_.lock()); @@ -77,15 +82,18 @@ struct TunnelLogger : public Logger to.flush(); } catch (...) { /* Write failed; that means that the other side is - gone. */ + gone, so stop sending it messages. Note that we + don't propagate the error, since logging must not + throw. The client's death will be detected + elsewhere (e.g. by `MonitorFdHup` or by the next + protocol read/write). */ state->canSendStderr = false; - throw; } } else state->pendingMsgs.push_back(s); } - void log(Verbosity lvl, std::string_view s) override + void log(Verbosity lvl, std::string_view s) noexcept override { if (lvl > verbosity) return; @@ -95,7 +103,7 @@ struct TunnelLogger : public Logger enqueueMsg(buf.s); } - void logEI(const ErrorInfo & ei) override + void logEI(const ErrorInfo & ei) noexcept override { if (ei.level > verbosity) return; @@ -148,7 +156,7 @@ struct TunnelLogger : public Logger ActivityType type, const std::string & s, const Fields & fields, - ActivityId parent) override + ActivityId parent) noexcept override { if (clientVersion.number < WorkerProto::Version::Number{1, 20}) { if (!s.empty()) @@ -161,7 +169,7 @@ struct TunnelLogger : public Logger enqueueMsg(buf.s); } - void stopActivity(ActivityId act) override + void stopActivity(ActivityId act) noexcept override { if (clientVersion.number < WorkerProto::Version::Number{1, 20}) return; @@ -170,7 +178,7 @@ struct TunnelLogger : public Logger enqueueMsg(buf.s); } - void result(ActivityId act, ResultType type, const Fields & fields) override + void result(ActivityId act, ResultType type, const Fields & fields) noexcept override { if (clientVersion.number < WorkerProto::Version::Number{1, 20}) return; @@ -180,22 +188,6 @@ struct TunnelLogger : public Logger } }; -struct TunnelSink : Sink -{ - Sink & to; - - TunnelSink(Sink & to) - : to(to) - { - } - - void operator()(std::string_view data) override - { - to << STDERR_WRITE; - writeString(data, to); - } -}; - struct TunnelSource : BufferedSource { Source & from; @@ -218,6 +210,8 @@ struct TunnelSource : BufferedSource } }; +} // namespace + struct ClientSettings { bool keepFailed; @@ -297,10 +291,9 @@ struct ClientSettings trusted || name == settings.getWorkerSettings().buildTimeout.name || name == settings.getWorkerSettings().maxSilentTime.name || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" - || (name == "builders" && value == "")) { + || name == loggerSettings.sessionId.name || (name == "builders" && value == "")) settings.set(name, value); - fileTransferSettings.set(name, value); - } else if (setSubstituters(settings.getWorkerSettings().substituters)) + else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( @@ -423,6 +416,9 @@ static void performOp( bool repairBool; conn.from >> repairBool; auto repair = RepairFlag{repairBool}; + auto provenance = conn.protoVersion.features.contains(WorkerProto::featureProvenance) + ? Provenance::from_json_str_optional(readString(conn.from)) + : nullptr; logger->startWork(); auto pathInfo = [&]() { @@ -448,8 +444,8 @@ static void performOp( assert(false); } // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. - auto path = - store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair); + auto path = store->addToStoreFromDump( + source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair, provenance); return store->queryPathInfo(path); }(); logger->stopWork(); @@ -510,7 +506,21 @@ static void performOp( logger->startWork(); { FramedSource source(conn.from); - store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); + auto expected = readNum(source); + for (uint64_t i = 0; i < expected; ++i) { + auto info = WorkerProto::Serialise::read( + *store, + WorkerProto::ReadConn{ + .from = source, + .version = conn.protoVersion.features.contains(WorkerProto::featureVersionedAddToStoreMultiple) + ? conn.protoVersion + : WorkerProto::Version{.number = {.major = 1, .minor = 16}}, + }); + info.ultimate = false; + EnsureRead wrapper{source, info.narSize}; + store->addToStore(info, wrapper, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); + wrapper.finish(); + } } logger->stopWork(); break; @@ -676,6 +686,15 @@ static void performOp( break; } + case WorkerProto::Op::AddTempRoots: { + auto paths = WorkerProto::Serialise::read(*store, rconn); + logger->startWork(); + store->addTempRoots(paths); + logger->stopWork(); + conn.to << 1; + break; + } + case WorkerProto::Op::AddPermRoot: { if (!trusted) throw Error( @@ -735,17 +754,36 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + } else { + auto paths = WorkerProto::Serialise::read(*store, rconn); + if (options.action != GCAction::gcDeleteSpecific && paths.empty()) + options.pathsToDelete = GCOptions::WholeStore{}; + else + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; + } conn.from >> options.ignoreLiveness >> options.maxFreed; + options.censor = !trusted; // obsolete fields readInt(conn.from); readInt(conn.from); readInt(conn.from); + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { + throw Error( + "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); + } + GCResults results; logger->startWork(); - if (options.ignoreLiveness) + if (options.ignoreLiveness && !getEnv("_NIX_IN_TEST").has_value()) throw Error("you are not allowed to ignore liveness"); auto & gcStore = require(*store); gcStore.collectGarbage(options, results); @@ -857,6 +895,37 @@ static void performOp( break; } + case WorkerProto::Op::QueryPathInfos: { + auto paths = WorkerProto::Serialise::read(*store, rconn); + logger->startWork(); + std::vector infos; + { + asio::io_context ctx; + std::exception_ptr ex; + asio::co_spawn( + ctx, + [&]() -> asio::awaitable { + co_await store->queryPathInfos( + paths, [&](std::vector>> results) { + for (auto & [path, info] : results) + if (info) + infos.push_back(*info); + }); + }, + [&](std::exception_ptr e) { ex = e; }); + ctx.run(); + if (ex) + std::rethrow_exception(ex); + } + logger->stopWork(); + /* Write the infos for the valid paths. Paths not reported are + invalid. */ + conn.to << infos.size(); + for (auto & info : infos) + WorkerProto::write(*store, wconn, info); + break; + } + case WorkerProto::Op::OptimiseStore: logger->startWork(); store->optimiseStore(); @@ -905,6 +974,9 @@ static void performOp( conn.from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = WorkerProto::Serialise>::read(*store, rconn); info.ca = ContentAddress::parseOpt(readString(conn.from)); + info.provenance = conn.protoVersion.features.contains(WorkerProto::featureProvenance) + ? Provenance::from_json_str_optional(readString(conn.from)) + : nullptr; conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; @@ -934,8 +1006,9 @@ static void performOp( logger->startWork(); - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); + EnsureRead wrapper{*source, info.narSize}; + store->addToStore(info, wrapper, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); + wrapper.finish(); logger->stopWork(); } @@ -957,14 +1030,8 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } @@ -972,19 +1039,16 @@ static void performOp( case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto info = store->queryRealisation(outputId); + auto ptr = store->queryRealisation(outputId); + std::optional info; + if (ptr) + info = *ptr; logger->stopWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - std::set outPaths; - if (info) - outPaths.insert(info->outPath); - WorkerProto::write(*store, wconn, outPaths); - } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); - } + /* Only return the new format because if we got past + `DrvOutput` serialization, we know that is what we're using. + */ + assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath)); + WorkerProto::write(*store, wconn, info); break; } @@ -1005,6 +1069,15 @@ static void performOp( break; } + case WorkerProto::Op::QueryActiveBuilds: { + logger->startWork(); + auto & activeBuildsStore = require(*store); + auto activeBuilds = activeBuildsStore.queryActiveBuilds(); + logger->stopWork(); + conn.to << nlohmann::json(activeBuilds).dump(); + break; + } + default: throw Error("invalid operation %1%", op); } @@ -1029,8 +1102,14 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted #endif /* Exchange the greeting. */ + auto localVersion = WorkerProto::latest; + if (recursive) + localVersion.features.insert(std::string{WorkerProto::featureDisableSetOptions}); + if (!experimentalFeatureSettings.isEnabled(Xp::Provenance)) + localVersion.features.erase(std::string(WorkerProto::featureProvenance)); + WorkerProto::BasicServerConnection conn; - conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, WorkerProto::latest); + conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, localVersion); if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix client version is too old"); @@ -1038,14 +1117,11 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted conn.to = std::move(to); conn.from = std::move(from); - auto tunnelLogger_ = std::make_unique(conn.to, conn.protoVersion); - auto tunnelLogger = tunnelLogger_.get(); - std::unique_ptr prevLogger_; - auto prevLogger = logger.get(); + auto tunnelLogger = new TunnelLogger(conn.to, conn.protoVersion); + auto prevLogger = logger; // FIXME if (!recursive) { - prevLogger_ = std::move(logger); - logger = std::move(tunnelLogger_); + logger = tunnelLogger; applyJSONLogger(); } diff --git a/src/libstore/darwin/build/darwin-derivation-builder.cc b/src/libstore/darwin/build/darwin-derivation-builder.cc new file mode 100644 index 000000000000..2e05f5858963 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.cc @@ -0,0 +1,313 @@ +#include "derivation-builder-impl.hh" +#include "darwin-derivation-builder.hh" + +#include +#include +#include +#include +#include +#include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int +sandbox_init_with_parameters(const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf); + +/* Darwin IPC structures and constants */ +#define IPCS_MAGIC 0x00000001 +#define IPCS_SHM_ITER 0x00000002 +#define IPCS_SEM_ITER 0x00000020 +#define IPCS_MSG_ITER 0x00000200 +#define IPCS_SHM_SYSCTL "kern.sysv.ipcs.shm" +#define IPCS_MSG_SYSCTL "kern.sysv.ipcs.msg" +#define IPCS_SEM_SYSCTL "kern.sysv.ipcs.sem" + +struct IpcsCommand +{ + uint32_t ipcs_magic; + uint32_t ipcs_op; + uint32_t ipcs_cursor; + uint32_t ipcs_datalen; + void * ipcs_data; +}; + +namespace nix { + +void DarwinDerivationBuilder::prepareSandbox() +{ + pathsInChroot = getPathsInSandbox(); +} + +void DarwinDerivationBuilder::setUser() +{ + DerivationBuilderImpl::setUser(); + + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useSandbox) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + StringSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + std::filesystem::path cur = i.first; + while (cur != "/") { + cur = cur.parent_path(); + ancestry.insert(cur.native()); + } + } + + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + std::filesystem::path cur = store.storeDir; + while (cur != "/") { + ancestry.insert(cur.native()); + cur = cur.parent_path(); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = store.printStorePath(i); + pathsInChroot.insert_or_assign(p, ChrootPath{.source = p}); + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be + * configurable */ + if (store.config->getLocalSettings().darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += +#include "sandbox-defaults.sb" + ; + + if (!derivationType.isSandboxed()) + sandboxProfile += +#include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + without file-write* allowed, access() incorrectly returns EPERM */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + + // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter. + // See https://github.com/NixOS/nix/issues/4119 + // We split our allow groups approximately at half the actual limit, 1 << 16 + const size_t breakpoint = sandboxProfile.length() + (1 << 14); + for (auto & i : pathsInChroot) { + + if (sandboxProfile.length() >= breakpoint) { + debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint); + sandboxProfile += ")\n(allow file-read* file-write* process-exec\n"; + } + + if (i.first != i.second.source) + throw Error( + "can't map %1% to %2%: mismatched impure paths not supported on Darwin", + PathFmt(i.first), + PathFmt(i.second.source)); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += drvOptions.additionalSandboxProfile; + } else + sandboxProfile += +#include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different + mechanisms to find temporary directories, so we want to open up a broader place for them to put their files, + if needed. */ + std::filesystem::path globalTmpDir = canonPath(defaultTempDir().native(), true); + + /* They don't like trailing slashes on subpath directives */ + std::string globalTmpDirStr = globalTmpDir.native(); + while (!globalTmpDirStr.empty() && globalTmpDirStr.back() == '/') + globalTmpDirStr.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_NIX_BUILD_TOP"); + sandboxArgs.push_back(tmpDir.native()); + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDirStr); + if (drvOptions.allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters( + sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull( + STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); + } + } +} + +void DarwinDerivationBuilder::execBuilder(const Strings & args, const Strings & envStrs) +{ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv.platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv.platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn( + NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +} + +void DarwinDerivationBuilder::cleanupSysVIPCForUser(uid_t uid) +{ + struct IpcsCommand ic; + size_t ic_size = sizeof(ic); + // IPC ids to cleanup + std::vector shm_ids, msg_ids, sem_ids; + + { + struct shmid_ds shm_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SHM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &shm_ds; + ic.ipcs_datalen = sizeof(shm_ds); + + while (true) { + memset(&shm_ds, 0, sizeof(shm_ds)); + + if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (shm_ds.shm_perm.uid == uid) { + int shmid = shmget(shm_ds.shm_perm._key, 0, 0); + if (shmid != -1) { + shm_ids.push_back(shmid); + } + } + } + } + + for (auto id : shm_ids) { + if (shmctl(id, IPC_RMID, NULL) == 0) + debug("removed shared memory segment with shmid %d", id); + } + + { + struct msqid_ds msg_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_MSG_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &msg_ds; + ic.ipcs_datalen = sizeof(msg_ds); + + while (true) { + memset(&msg_ds, 0, sizeof(msg_ds)); + + if (sysctlbyname(IPCS_MSG_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (msg_ds.msg_perm.uid == uid) { + int msgid = msgget(msg_ds.msg_perm._key, 0); + if (msgid != -1) { + msg_ids.push_back(msgid); + } + } + } + } + + for (auto id : msg_ids) { + if (msgctl(id, IPC_RMID, NULL) == 0) + debug("removed message queue with msgid %d", id); + } + + { + struct semid_ds sem_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SEM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &sem_ds; + ic.ipcs_datalen = sizeof(sem_ds); + + while (true) { + memset(&sem_ds, 0, sizeof(sem_ds)); + + if (sysctlbyname(IPCS_SEM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (sem_ds.sem_perm.uid == uid) { + int semid = semget(sem_ds.sem_perm._key, 0, 0); + if (semid != -1) { + sem_ids.push_back(semid); + } + } + } + } + + for (auto id : sem_ids) { + if (semctl(id, 0, IPC_RMID) == 0) + debug("removed semaphore with semid %d", id); + } +} + +void DarwinDerivationBuilder::killSandbox(bool getStats) +{ + DerivationBuilderImpl::killSandbox(getStats); + if (buildUser) { + auto uid = buildUser->getUID(); + cleanupSysVIPCForUser(uid); + } +} + +} // namespace nix diff --git a/src/libstore/darwin/build/darwin-derivation-builder.hh b/src/libstore/darwin/build/darwin-derivation-builder.hh new file mode 100644 index 000000000000..260339c5c577 --- /dev/null +++ b/src/libstore/darwin/build/darwin-derivation-builder.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct DarwinDerivationBuilder : DerivationBuilderImpl +{ + PathsInChroot pathsInChroot; + + /** + * Whether full sandboxing is enabled. Note that macOS builds + * always have *some* sandboxing (see sandbox-minimal.sb). + */ + bool useSandbox; + + DarwinDerivationBuilder( + LocalStore & store, + std::shared_ptr miscMethods, + DerivationBuilderParams params, + bool useSandbox) + : DerivationBuilderImpl(store, miscMethods, std::move(params)) + , useSandbox(useSandbox) + { + } + + void prepareSandbox() override; + + void setUser() override; + + void execBuilder(const Strings & args, const Strings & envStrs) override; + + /** + * Cleans up all System V IPC objects owned by the specified user. + * + * On Darwin, IPC objects (shared memory segments, message queues, and semaphore) + * can persist after the build user's processes are killed, since there are no IPC namespaces + * like on Linux. This can exhaust kernel IPC limits over time. + * + * Uses sysctl to enumerate and remove all IPC objects owned by the given UID. + */ + void cleanupSysVIPCForUser(uid_t uid); + + void killSandbox(bool getStats) override; +}; + +} // namespace nix diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/darwin/build/sandbox-defaults.sb similarity index 100% rename from src/libstore/unix/build/sandbox-defaults.sb rename to src/libstore/darwin/build/sandbox-defaults.sb diff --git a/src/libstore/unix/build/sandbox-minimal.sb b/src/libstore/darwin/build/sandbox-minimal.sb similarity index 100% rename from src/libstore/unix/build/sandbox-minimal.sb rename to src/libstore/darwin/build/sandbox-minimal.sb diff --git a/src/libstore/unix/build/sandbox-network.sb b/src/libstore/darwin/build/sandbox-network.sb similarity index 100% rename from src/libstore/unix/build/sandbox-network.sb rename to src/libstore/darwin/build/sandbox-network.sb diff --git a/src/libstore/darwin/meson.build b/src/libstore/darwin/meson.build new file mode 100644 index 000000000000..72d94baac2be --- /dev/null +++ b/src/libstore/darwin/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/darwin-derivation-builder.cc') diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5208440c7d94..b3d4261b5464 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -6,7 +6,6 @@ #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/variant-wrapper.hh" #include #include @@ -73,7 +72,8 @@ template using OutputChecks = DerivationOptions::OutputChecks; template -using OutputChecksVariant = std::variant, std::map>>; +using OutputChecksVariant = + std::variant, std::map, std::less<>>>; DerivationOptions derivationOptionsFromStructuredAttrs( const StoreDirConfig & store, @@ -124,7 +124,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( { DerivationOptions defaults = {}; - std::map placeholders; + std::map> placeholders; if (mockXpSettings.isEnabled(Xp::CaDerivations)) { /* Initialize placeholder map from inputDrvs */ auto initPlaceholders = [&](this const auto & initPlaceholders, @@ -156,16 +156,27 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } } + /* Extract the placeholder key from a path that may have a subpath + appended (e.g. `/HASH/foo` → `/HASH`), mirroring how + `StoreDirConfig::toStorePath` strips subpaths from store paths. */ + auto findPlaceholder = [&](std::string_view pathS) -> const SingleDerivedPath::Built * { + auto slash = pathS.find('/', 1); + auto key = pathS.substr(0, slash); + if (auto it = placeholders.find(key); it != placeholders.end()) + return &it->second; + return nullptr; + }; + auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; else return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; }; auto parseRef = [&](const std::string & pathS) -> DrvRef { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; if (store.isStorePath(pathS)) return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; else @@ -206,7 +217,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( if (parsed) { auto & structuredAttrs = parsed->structuredAttrs; - std::map> res; + std::map, std::less<>> res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { for (auto & [outputName, output_] : getObject(*outputChecks)) { auto & output = getObject(output_); @@ -265,7 +276,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .unsafeDiscardReferences = [&] { - std::map res; + std::map> res; if (parsed) { if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { @@ -298,7 +309,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .exportReferencesGraph = [&] { - std::map> ret; + std::map, std::less<>> ret; if (parsed) { auto * e = get(parsed->structuredAttrs, "exportReferencesGraph"); @@ -444,9 +455,10 @@ std::optional> tryResolve( }; // Helper function to resolve exportReferencesGraph using functional style - auto tryResolveExportReferencesGraph = [&](const std::map> & exportGraph) - -> std::optional>> { - std::map> resolved; + auto tryResolveExportReferencesGraph = + [&](const std::map, std::less<>> & exportGraph) + -> std::optional, std::less<>>> { + std::map, std::less<>> resolved; for (const auto & [name, inputPaths] : exportGraph) { std::set resolvedPaths; for (const auto & inputPath : inputPaths) { @@ -466,19 +478,20 @@ std::optional> tryResolve( [&](const DerivationOptions::OutputChecks & checks) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { + std::map::OutputChecks, std::less<>>>> { auto resolved = tryResolveOutputChecks(checks); if (!resolved) return std::nullopt; return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(*resolved); + std::map::OutputChecks, std::less<>>>(*resolved); }, - [&](const std::map::OutputChecks> & checksMap) + [&](const std::map::OutputChecks, std::less<>> & + checksMap) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { - std::map::OutputChecks> resolvedMap; + std::map::OutputChecks, std::less<>>>> { + std::map::OutputChecks, std::less<>> resolvedMap; for (const auto & [outputName, checks] : checksMap) { auto resolved = tryResolveOutputChecks(checks); if (!resolved) @@ -487,7 +500,7 @@ std::optional> tryResolve( } return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(resolvedMap); + std::map::OutputChecks, std::less<>>>(resolvedMap); }}, drvOptions.outputChecks); @@ -525,21 +538,22 @@ namespace nlohmann { using namespace nix; -DerivationOptions adl_serializer>::from_json(const json & json_) +template +static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); return { - .outputChecks = [&]() -> OutputChecksVariant { + .outputChecks = [&]() -> OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); auto forAllOutputsOpt = get(outputChecks, "forAllOutputs"); auto perOutputOpt = get(outputChecks, "perOutput"); if (forAllOutputsOpt && !perOutputOpt) { - return static_cast>(*forAllOutputsOpt); + return static_cast>(*forAllOutputsOpt); } else if (perOutputOpt && !forAllOutputsOpt) { - return static_cast>>(*perOutputOpt); + return static_cast, std::less<>>>(*perOutputOpt); } else { throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); } @@ -561,17 +575,17 @@ DerivationOptions adl_serializer>::to_json( - json & json, const DerivationOptions & o) +template +static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) { json["outputChecks"] = std::visit( overloaded{ - [&](const OutputChecks & checks) { + [&](const OutputChecks & checks) { nlohmann::json outputChecks; outputChecks["forAllOutputs"] = checks; return outputChecks; }, - [&](const std::map> & checksPerOutput) { + [&](const std::map, std::less<>> & checksPerOutput) { nlohmann::json outputChecks; outputChecks["perOutput"] = checksPerOutput; return outputChecks; @@ -594,7 +608,8 @@ void adl_serializer>::to_json( json["allowSubstitutes"] = o.allowSubstitutes; } -OutputChecks adl_serializer>::from_json(const json & json_) +template +static OutputChecks outputChecksFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); @@ -602,16 +617,15 @@ OutputChecks adl_serializer>: .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), - .allowedReferences = - ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), + .allowedReferences = ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = valueAt(json, "disallowedReferences"), - .allowedRequisites = - ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), + .allowedRequisites = ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), .disallowedRequisites = valueAt(json, "disallowedRequisites"), }; } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +template +static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -622,4 +636,45 @@ void adl_serializer>::to_json(json & json, const json["disallowedRequisites"] = c.disallowedRequisites; } +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json( + json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + } // namespace nlohmann diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ab969a089f57..b4f1f5a27fa2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,14 +1,13 @@ #include "nix/store/derivations.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/split.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/util/strings-inline.hh" #include "nix/util/json-utils.hh" +#include "nix/store/async-path-writer.hh" #include #include @@ -19,6 +18,10 @@ namespace nix { using namespace std::literals::string_view_literals; +BasicDerivation::~BasicDerivation() {} + +Derivation::~Derivation() {} + std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { @@ -132,7 +135,8 @@ StorePath computeStorePath(const StoreDirConfig & store, const Derivation & drv) return path; } -StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) +StorePath +Store::writeDerivation(const Derivation & drv, RepairFlag repair, std::shared_ptr provenance) { auto [suffix, contents, references, path] = infoForDerivation(*this, drv); @@ -153,12 +157,26 @@ StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, - repair); + repair, + provenance); assert(path2 == path); return path; } +StorePath Store::writeDerivation( + AsyncPathWriter & asyncPathWriter, + const Derivation & drv, + RepairFlag repair, + std::shared_ptr provenance) +{ + auto references = drv.inputSrcs; + for (auto & i : drv.inputDrvs.map) + references.insert(i.first); + return asyncPathWriter.addPath( + drv.unparse(*this, false), std::string(drv.name) + drvExtension, references, repair, provenance); +} + namespace { /** * This mimics std::istream to some extent. We use this much smaller implementation @@ -225,6 +243,7 @@ static BackedStringView parseString(StringViewStream & str) size_t start = 0; size_t end = str.remaining.size(); const auto data = str.remaining.data(); + bool foundClose = false; while (start < end) { auto idx = str.remaining.find('"', start); if (idx == std::string_view::npos) { @@ -235,10 +254,13 @@ static BackedStringView parseString(StringViewStream & str) ; if ((idx - pos) % 2 == 0) { // even number of backslashes end = idx; + foundClose = true; break; } start = idx + 1; } + if (!foundClose) + throw FormatError("unterminated string in derivation"); start = 0; const auto content = str.remaining.substr(start, end); @@ -748,7 +770,7 @@ std::string Derivation::unparse( s += ",["sv; first = true; - auto unparseEnv = [&](const StringPairs atermEnv) { + auto unparseEnv = [&](const StringPairs & atermEnv) { for (auto & i : atermEnv) { if (first) first = false; @@ -861,13 +883,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -890,12 +913,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -905,54 +926,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { - const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; } - } - - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); + const auto & res = pathDerivationModulo(store, drvPath); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; + } } - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1102,26 +1135,6 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) -{ - drv.applyRewrites(rewrites); - - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); - for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; - } - } -} - bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ @@ -1168,7 +1181,7 @@ std::optional Derivation::tryResolve(Store & store, Store * eva } static bool tryResolveInput( - Store & store, + const StoreDirConfig & store, StorePathSet & inputSrcs, StringMap & inputRewrites, const DownstreamPlaceholder * placeholderOpt, @@ -1233,9 +1246,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); - return resolved; + Derivation resolved2{std::move(resolved)}; + + resolved2.fillInOutputPaths(store); + + return resolved2; } /** @@ -1262,7 +1279,15 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { - std::optional hashesModulo; + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; for (auto & [outputName, output] : drv.outputs) { auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) { @@ -1299,65 +1324,64 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string } }; auto hash = [&](const Output & outputVariant) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, drv, true); - } - switch (hashesModulo->kind) { - case DrvHash::Kind::Regular: { - auto h = get(hashesModulo->hashes, outputName); - if (!h) - throw Error("derivation produced no hash for output '%s'", outputName); - auto outPath = store.makeOutputPath(outputName, *h, drvName); - - if constexpr (std::is_same_v) { - if (outputVariant.path == outPath) { - return; // Correct case - } - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be '%s'", - store.printStorePath(outputVariant.path), - store.printStorePath(outPath)); - } else if constexpr (std::is_same_v) { - if constexpr (fillIn) - /* Fill in output path for Deferred - outputs */ - output = DerivationOutput::InputAddressed{ - .path = outPath, - }; - else - /* Validation mode: deferred outputs - should have been filled in */ - warn( - "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", - store.printStorePath(outPath)); - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - envHasRightPath(outPath); - break; - } - case DrvHash::Kind::Deferred: - if constexpr (std::is_same_v) { - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be deferred", - store.printStorePath(outputVariant.path)); - } else if constexpr (std::is_same_v) { - /* Correct: Deferred output with Deferred - hash kind. */ - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - break; - } + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drvName); + + if constexpr (std::is_same_v) { + if (outputVariant.path == outPath) { + envHasRightPath(outPath); + return; // Correct case + } + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be '%s'", + store.printStorePath(outputVariant.path), + store.printStorePath(outPath)); + } else if constexpr (std::is_same_v) { + if constexpr (fillIn) { + /* Fill in output path for Deferred outputs */ + output = DerivationOutput::InputAddressed{ + .path = outPath, + }; + envHasRightPath(outPath); + } else { + /* Validation mode: deferred outputs + should have been filled in */ + warn( + "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", + store.printStorePath(outPath)); + } + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed (or deferred waiting to be). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + if constexpr (std::is_same_v) { + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be deferred", + store.printStorePath(outputVariant.path)); + } else if constexpr (std::is_same_v) { + /* Correct: Deferred output with Deferred hash kind. */ + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + }, + hashModulo().raw); }; std::visit( overloaded{ @@ -1524,32 +1548,52 @@ adl_serializer::from_json(const json & _json, const Experiment } } -void adl_serializer::to_json(json & res, const Derivation & d) +static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +{ + res = nlohmann::json::array(); + for (auto & input : inputSrcs) + res.emplace_back(input); +} + +static void basicDerivationToJson(json & res, const BasicDerivation & d) { res = nlohmann::json::object(); res["name"] = d.name; - res["version"] = expectedJsonVersionDerivation; { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : d.outputs) { + for (auto & [outputName, output] : d.outputs) outputsObj[outputName] = output; - } } + res["system"] = d.platform; + res["builder"] = d.builder; + res["args"] = d.args; + res["env"] = d.env; + + if (d.structuredAttrs) + res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; +} + +void adl_serializer::to_json(json & res, const BasicDerivation & d) +{ + basicDerivationToJson(res, d); + + inputSrcsToJson(res["inputs"], d.inputSrcs); +} + +void adl_serializer::to_json(json & res, const Derivation & d) +{ + basicDerivationToJson(res, d); + { auto & inputsObj = res["inputs"]; inputsObj = nlohmann::json::object(); - { - auto & inputsList = inputsObj["srcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); @@ -1565,28 +1609,21 @@ void adl_serializer::to_json(json & res, const Derivation & d) auto & inputDrvsObj = inputsObj["drvs"]; inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } } - - res["system"] = d.platform; - res["builder"] = d.builder; - res["args"] = d.args; - res["env"] = d.env; - - if (d.structuredAttrs) - res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) { - using nlohmann::detail::value_t; - - Derivation res; - - auto & json = getObject(_json); + auto arr = getArray(inputSrcsJson); + for (auto & input : arr) + inputSrcs.insert(input); +} +static void basicDerivationFromJson( + const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) +{ res.name = getString(valueAt(json, "name")); { @@ -1608,13 +1645,50 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + + auto envJson = valueAt(json, "env"); + try { + res.env = getStringMap(envJson); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + + if (auto structuredAttrs = get(json, "structuredAttrs")) + res.structuredAttrs = StructuredAttrs{*structuredAttrs}; +} + +BasicDerivation +adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + BasicDerivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + + try { + inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + + return res; +} + +Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + Derivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + try { auto inputsObj = getObject(valueAt(json, "inputs")); try { - auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); + inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); } catch (Error & e) { e.addTrace({}, "while reading key 'srcs'"); throw; @@ -1645,21 +1719,6 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); - try { - res.env = getStringMap(envJson); - } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); - throw; - } - - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; - return res; } diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 3d799ab1c583..684c4462fbf8 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -48,6 +48,34 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single return initIter(k); } +template +void DerivedPathMap::removeSlot(const SingleDerivedPath & k, fun callback) +{ + auto removeIter = [&map = + map](this auto & self, const SingleDerivedPath & k, fun onNode) -> void { + std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { + if (auto it = map.find(bo.path); it != map.end() && !onNode(it->second)) + map.erase(it); + }, + [&](const SingleDerivedPath::Built & bfd) { + self(*bfd.drvPath, [&](ChildNode & parent) -> bool { + auto it = parent.childMap.find(bfd.output); + if (it == parent.childMap.end()) + return !parent.value.empty() || !parent.childMap.empty(); + if (!onNode(it->second)) + parent.childMap.erase(it); + return !parent.value.empty() || !parent.childMap.empty(); + }); + }, + }, + k.raw()); + }; + + removeIter(k, [&](ChildNode & node) -> bool { return callback(node) || !node.childMap.empty(); }); +} + } // namespace nix // instantiations diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 251e112514e1..131674aa5595 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,11 +1,9 @@ #include "nix/store/derived-path.hh" #include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/comparator.hh" #include "nix/util/json-utils.hh" -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 943873119ca5..3c4de14edecd 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -31,6 +35,8 @@ namespace { class WholeStoreViewAccessor : public SourceAccessor { + void anchor() override {}; + using BaseName = std::string; /** @@ -63,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -101,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } @@ -126,6 +135,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -215,7 +228,9 @@ struct DummyStoreImpl : DummyStore if (info.path.isDerivation()) { warn("back compat supporting `addToStore` for inserting derivations in dummy store"); writeDerivation( - parseDerivation(*this, accessor->readFile(CanonPath::root), Derivation::nameFromPath(info.path))); + parseDerivation(*this, accessor->readFile(CanonPath::root), Derivation::nameFromPath(info.path)), + repair, + info.provenance); return; } @@ -232,11 +247,12 @@ struct DummyStoreImpl : DummyStore StorePath addToStoreFromDump( Source & source, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, - ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, - HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair, + std::shared_ptr provenance) override { if (isDerivation(name)) throw Error("Do not insert derivation into dummy store with `addToStoreFromDump`"); @@ -284,6 +300,7 @@ struct DummyStoreImpl : DummyStore std::move(narHash.first)); info.narSize = narHash.second.value(); + info.provenance = provenance; auto path = info.path; auto accessor = make_ref(std::move(*temp)); @@ -299,7 +316,8 @@ struct DummyStoreImpl : DummyStore return path; } - StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair) override + StorePath + writeDerivation(const Derivation & drv, RepairFlag repair, std::shared_ptr provenance) override { auto drvPath = nix::computeStorePath(*this, drv); @@ -307,6 +325,7 @@ struct DummyStoreImpl : DummyStore if (config->readOnly) unsupported("writeDerivation"); derivations.insert({drvPath, drv}); + // FIXME: record provenance } return drvPath; @@ -330,7 +349,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, output); }); } @@ -339,7 +358,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(std::make_shared(it->second)); @@ -378,6 +397,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); @@ -436,11 +457,7 @@ ref adl_serializer>::from_json(const json & json) for (auto & [k1, v2] : getObject(v)) { UnkeyedRealisation realisation = v2; res->buildTrace.insert_or_visit( - { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), - {{k1, realisation}}, - }, - [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + {StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); } } return res; @@ -473,7 +490,7 @@ void adl_serializer::to_json(json & json, const DummyStore & val) auto obj = json::object(); val.buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = v2; }); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b1c61626c8c7..8dec3fcd5080 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -4,92 +4,173 @@ #include "nix/util/archive.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" - -#include +#include "nix/store/worker-protocol.hh" namespace nix { -static void exportPath(Store & store, const StorePath & path, Sink & sink) -{ - auto info = store.queryPathInfo(path); - - HashSink hashSink(HashAlgorithm::SHA256); - TeeSink teeSink(sink, hashSink); - - store.narFromPath(path, teeSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashSink.currentHash().hash; - if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) - throw Error( - "hash of path '%s' has changed from '%s' to '%s'!", - store.printStorePath(path), - info->narHash.to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)); - - teeSink << exportMagic << store.printStorePath(path); - CommonProto::write(store, CommonProto::WriteConn{.to = teeSink}, info->references); - teeSink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; -} +static const uint32_t exportMagicV1 = 0x4558494e; +static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2' -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink) +static WorkerProto::Version exportProtoVersion{ + .number = + { + .major = 1, + .minor = 16, + }, +}; + +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) { auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - for (auto & path : sorted) { - sink << 1; - exportPath(store, path, sink); + auto dumpNar = [&](const ValidPathInfo & info) { + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(sink, hashSink); + + store.narFromPath(info.path, teeSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashSink.currentHash().hash; + if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) + throw Error( + "hash of path '%s' has changed from '%s' to '%s'!", + store.printStorePath(info.path), + info.narHash.to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)); + }; + + switch (version) { + + case 1: + for (auto & path : sorted) { + sink << 1; + auto info = store.queryPathInfo(path); + dumpNar(*info); + sink << exportMagicV1 << store.printStorePath(path); + CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references); + sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; + } + sink << 0; + break; + + case 2: + sink << exportMagicV2; + + for (auto & path : sorted) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + sink << 1; + auto info = store.queryPathInfo(path); + // FIXME: move to CommonProto? + WorkerProto::Serialise::write( + store, + WorkerProto::WriteConn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}, + *info); + dumpNar(*info); + } + + sink << 0; + break; + + default: + throw Error("unsupported nario version %d", version); } - - sink << 0; } StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) { StorePaths res; - while (true) { - auto n = readNum(source); - if (n == 0) - break; - if (n != 1) - throw Error("input doesn't look like something created by 'nix-store --export'"); - - /* Extract the NAR from the source. */ + + auto version = readNum(source); + + /* Note: nario version 1 lacks an explicit header. The first + integer denotes whether a store path follows or not. So look + for 0 or 1. */ + switch (version) { + + case 0: + /* Empty version 1 nario, nothing to do. */ + break; + + case 1: { + /* Reuse a string buffer to avoid kernel overhead allocating + memory for large strings. */ StringSink saved; - TeeSource tee{source, saved}; - NullFileSystemObjectSink ether; - parseDump(ether, tee); - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); + /* Non-empty version 1 nario. */ + while (true) { + /* Extract the NAR from the source. */ + saved.s.clear(); + TeeSource tee{source, saved}; + NullFileSystemObjectSink ether; + parseDump(ether, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagicV1) + throw Error("nario cannot be imported; wrong format"); + + auto path = store.parseStorePath(readString(source)); + + Activity act(*logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(path))); + + auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); + auto deriver = readString(source); + + // Ignore optional legacy signature. + if (readInt(source) == 1) + readString(source); + + if (!store.isValidPath(path)) { + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + + ValidPathInfo info{path, {store, narHash}}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); + + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + store.addToStore(info, source2, NoRepair, checkSigs); + } + + res.push_back(path); + + auto n = readNum(source); + if (n == 0) + break; + if (n != 1) + throw Error("input doesn't look like a nario"); + } + break; + } - auto path = store.parseStorePath(readString(source)); + case exportMagicV2: + while (true) { + auto n = readNum(source); + if (n == 0) + break; + if (n != 1) + throw Error("input doesn't look like a nario"); - // Activity act(*logger, lvlInfo, "importing path '%s'", info.path); + auto info = WorkerProto::Serialise::read( + store, WorkerProto::ReadConn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}); - auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); - auto deriver = readString(source); - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + Activity act( + *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); - ValidPathInfo info{path, {store, narHash}}; - if (deriver != "") - info.deriver = store.parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); + EnsureRead wrapper{source, info.narSize}; + store.addToStore(info, wrapper, NoRepair, checkSigs); - // Ignore optional legacy signature. - if (readInt(source) == 1) - readString(source); + res.push_back(info.path); + } - // Can't use underlying source, which would have been exhausted - auto source = StringSource(saved.s); - store.addToStore(info, source, NoRepair, checkSigs); + break; - res.push_back(info.path); + default: + throw Error("input doesn't look like a nario"); } return res; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index ca0bdb46b8df..80f6a6be0735 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,12 +1,12 @@ #include "nix/store/filetransfer.hh" +#include "nix/store/filetransfer-impl.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" -#include "nix/store/store-api.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" +#include "nix/util/util.hh" -#include "store-config-private.hh" #include "nix/store/s3-url.hh" #include #if NIX_WITH_AWS_AUTH @@ -22,38 +22,103 @@ #include -#include +#include +#include +#include #include #include #include #include +#include #include namespace nix { -const unsigned int RETRY_TIME_MS_DEFAULT = 250; -const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; +namespace { + +enum struct HttpStatus : long { + Ok = 200, + Created = 201, + NoContent = 204, + PartialContent = 206, + NotModified = 304, + Unauthorized = 401, + Forbidden = 403, + NotFound = 404, + ProxyAuthRequired = 407, + RequestTimeout = 408, + Gone = 410, + TooManyRequests = 429, + NotImplemented = 501, + ServiceUnavailable = 503, + HttpVersionNotSupported = 505, + NetworkAuthRequired = 511, +}; -std::filesystem::path FileTransferSettings::getDefaultSSLCertFile() +constexpr bool operator==(long lhs, HttpStatus rhs) noexcept { + return lhs == static_cast(rhs); +} + +} // namespace + +std::chrono::milliseconds computeRetryDelayMs(const RetryDelayParams & p, std::mt19937 & rng) +{ + uint32_t backoff = clampedExponential(p.baseMs, p.attempt, p.ceilMs); + + // Retry-After is a hard minimum — the server explicitly asked us to wait + // at least this long. ceilMs caps the backoff algorithm, not the server's + // signal: retrying before the server is ready just burns an attempt. + uint32_t floor = p.retryAfterMs.value_or(0); + + if (!p.jitter) + return std::chrono::milliseconds(std::max(floor, backoff)); + + // Jitter spreads retries over [floor, floor+backoff] so that concurrent + // clients receiving the same Retry-After don't all retry simultaneously. + // Saturating add: clamp if floor + backoff would overflow uint32_t. + constexpr auto u32max = std::numeric_limits::max(); + uint32_t ceiling = (backoff > u32max - floor) ? u32max : floor + backoff; + if (ceiling <= floor) + return std::chrono::milliseconds(floor); + + return std::chrono::milliseconds(std::uniform_int_distribution(floor, ceiling)(rng)); +} + +std::optional FileTransferSettings::getDefaultSSLCertFile() +{ + /* Windows has no notion of a default location for the certificate bundles. + Instead we use CURLSSLOPT_NATIVE_CA by default. */ +#ifndef _WIN32 for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) if (pathAccessible(fn)) return fn; - return ""; +#endif + return std::nullopt; } +void FileTransferSettings::anchor() {} + FileTransferSettings::FileTransferSettings() { - auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (sslOverride != "") - caFile = sslOverride; + std::optional sslOverride = + getEnvOs(OS_STR("NIX_SSL_CERT_FILE")) + .or_else([] { return getEnvOs(OS_STR("SSL_CERT_FILE")); }) + .and_then([](OsString s) -> std::optional { + return s.empty() ? std::nullopt : std::optional{std::move(s)}; + }) + .transform([](OsString s) { return AbsolutePath{std::filesystem::path{std::move(s)}}; }); + if (sslOverride) + caFile = *sslOverride; } FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); +FileTransfer::~FileTransfer() {} + namespace { using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>; @@ -70,12 +135,21 @@ struct curlMultiError final : CloneableError } }; +/* Check if the linked libcurl was built with HTTP3 support. */ +bool curlSupportsHttp3() +{ + const auto * info = ::curl_version_info(CURLVERSION_NOW); + return info && (info->features & CURL_VERSION_HTTP3); +} + } // namespace struct curlFileTransfer : public FileTransfer { const FileTransferSettings & settings; + const bool http3Supported = curlSupportsHttp3(); + curlMulti curlm; std::random_device rd; @@ -93,7 +167,7 @@ struct curlFileTransfer : public FileTransfer char errbuf[CURL_ERROR_SIZE]; std::string statusMsg; - unsigned int attempt = 0; + uint32_t attempt = 0; /* Don't start this download until the specified time point has been reached. */ @@ -126,16 +200,36 @@ struct curlFileTransfer : public FileTransfer */ bool acceptRanges:1 = false; + /** + * When retrying, whether to request a range. + */ + bool requestRange:1 = false; + /** * Whether the response has a non-trivial (not "identity") Content-Encoding. */ bool hasContentEncoding:1 = false; + /** + * Server-provided minimum retry delay, parsed from the `Retry-After` + * response header. Reset on each new HTTP status line, and consumed + * (cleared) by maybeRetry() so it applies to at most one retry attempt. + */ + std::optional retryAfterMs; + + curl_off_t bytesReceived = 0; + curl_off_t writtenToSink = 0; std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); - inline static const std::set successfulStatuses{200, 201, 204, 206, 304, 0 /* other protocol */}; + inline static const std::set successfulStatuses{ + static_cast(HttpStatus::Ok), + static_cast(HttpStatus::Created), + static_cast(HttpStatus::NoContent), + static_cast(HttpStatus::PartialContent), + static_cast(HttpStatus::NotModified), + 0 /* other protocol */}; /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() @@ -175,6 +269,18 @@ struct curlFileTransfer : public FileTransfer /* Only write data to the sink if this is a successful response. */ if (successfulStatuses.count(httpStatus)) { + + auto prevReceived = bytesReceived; + bytesReceived += data.size(); + + /* Discard data that we've already received and sent to the sink in a previous try. */ + if (httpStatus != 206 && prevReceived < writtenToSink) { + if (writtenToSink - prevReceived >= (curl_off_t) data.size()) { + return; + } + data = data.substr(writtenToSink - prevReceived); + } + writtenToSink += data.size(); PauseTransfer needsPause = this->request.dataCallback(data); if (needsPause == PauseTransfer::Yes) { @@ -206,13 +312,8 @@ struct curlFileTransfer : public FileTransfer curl_multi_remove_handle(fileTransfer.curlm.get(), req); curl_easy_cleanup(req); } - try { - if (!done && enqueued) - fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); - } catch (...) { - ignoreExceptionInDestructor(); - } + if (!done && enqueued) + failInterruptedOrCancelled(); } void failEx(std::exception_ptr ex) noexcept @@ -225,7 +326,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -238,6 +339,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -289,7 +402,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -299,6 +412,8 @@ struct curlFileTransfer : public FileTransfer statusMsg = trim(match.str(1)); acceptRanges = false; hasContentEncoding = false; + retryAfterMs = std::nullopt; + bytesReceived = 0; appendCurrentUrl(); } else { @@ -315,7 +430,7 @@ struct curlFileTransfer : public FileTransfer data. */ long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - if (result.etag == request.expectedETag && httpStatus == 200) { + if (result.etag == request.expectedETag && httpStatus == HttpStatus::Ok) { debug("shutting down on 200 HTTP response with expected ETag"); return 0; } @@ -338,6 +453,19 @@ struct curlFileTransfer : public FileTransfer else debug("got invalid link header '%s'", value); } + + else if (name == "retry-after") { + auto value = trim(line.substr(i + 1)); + // RFC 7231 §7.1.3: Retry-After = HTTP-date / delay-seconds. + if (auto seconds = string2Int(value)) { + retryAfterMs = saturateMs(std::chrono::seconds{*seconds}); + } else if (time_t date = curl_getdate(requireCString(value), nullptr); date != -1) { + time_t now = time(nullptr); + retryAfterMs = saturateMs(std::chrono::seconds{date > now ? date - now : 0}); + } else { + debug("ignoring unparseable Retry-After header: '%s'", value); + } + } } } return realSize; @@ -364,8 +492,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -477,6 +605,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_reset(req); + bytesReceived = 0; + if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); @@ -488,7 +618,7 @@ struct curlFileTransfer : public FileTransfer Skip for uploads (Accept-Encoding is meaningless when sending data) and when resuming from an offset (byte ranges don't work with compressed content). */ - if (writtenToSink == 0 && !request.data) + if (!requestRange && !request.data) /* Empty string means to enable all supported (that libcurl has been linked to support) encodings. */ curl_easy_setopt(req, CURLOPT_ACCEPT_ENCODING, ""); @@ -499,12 +629,15 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt( req, CURLOPT_USERAGENT, - ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + " DeterminateNix/" + determinateNixVersion + (fileTransfer.settings.userAgentSuffix != "" ? " " + fileTransfer.settings.userAgentSuffix.get() : "")) .c_str()); curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); - if (fileTransfer.settings.enableHttp2) + /* Enable HTTP3 only on user config and linked libcurl have support, o.w. fall back. */ + if (fileTransfer.settings.enableHttp3 && fileTransfer.http3Supported) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_3); + else if (fileTransfer.settings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -550,23 +683,38 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } + /* Note: libcurl copies string arguments, so temporaries from + .string().c_str() are safe. See the comment near CURLOPT_SSLKEY below. */ if (auto & caFile = fileTransfer.settings.caFile.get()) - curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, caFile->string().c_str()); +#ifdef _WIN32 + /* Use native windows certificate store when the option is not specified explicitly. */ + else + curl_easy_setopt(req, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); +#endif #if !defined(_WIN32) curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); #endif curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransfer.settings.connectTimeout.get()); + /* Enable TCP keepalive to detect dead connections and server closures. + Probes every 30s to catch network failures and idle timeouts early. */ + curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 30L); + curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 30L); + /* Don't reuse idle connections older than 90s. */ + curl_easy_setopt(req, CURLOPT_MAXAGE_CONN, 90L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransfer.settings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().string().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); - if (writtenToSink) + if (requestRange) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); /* Note that the underlying strings get copied by libcurl, so the path -> string conversion is ok: @@ -618,14 +766,12 @@ struct curlFileTransfer : public FileTransfer { auto finishTime = std::chrono::steady_clock::now(); - auto retryTimeMs = request.baseRetryTimeMs; - auto httpStatus = getHTTPStatus(); debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -635,19 +781,19 @@ struct curlFileTransfer : public FileTransfer if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; - httpStatus = 304; + httpStatus = std::to_underlying(HttpStatus::NotModified); } if (callbackException) failEx(callbackException); else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { - result.cached = httpStatus == 304; + result.cached = (httpStatus == HttpStatus::NotModified); // In 2021, GitHub responds to If-None-Match with 304, // but omits ETag. We just use the If-None-Match etag // since 304 implies they are the same. - if (httpStatus == 304 && result.etag == "") + if (httpStatus == HttpStatus::NotModified && result.etag == "") result.etag = request.expectedETag; curl_off_t dlSize = 0; @@ -661,20 +807,57 @@ struct curlFileTransfer : public FileTransfer // We treat most errors as transient, but won't retry when hopeless Error err = Transient; - if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { + // S3 returns certain retryable errors as HTTP 400/500/503 with XML error codes. + // These take precedence over the generic HTTP status handling below. + // Only parse the response body on status codes where S3 XML errors can appear. + static constexpr std::array s3RetryableErrors{{ + "IncompleteBody", // HTTP 400 - network issue + "InternalError", // HTTP 500 - S3 internal failure + "InternalFailure", // HTTP 500 - alias for InternalError + "InternalServerError", // HTTP 500 - alias for InternalError + "RequestExpired", // HTTP 400 - clock skew / slow upload + "RequestTimeout", // HTTP 400 - stale connection reuse + "RequestTimeTooSkewed", // HTTP 403 - clock drift + "RequestThrottled", // HTTP 400 - throttling variant + "SlowDown", // HTTP 503 - throttling + "ServiceUnavailable", // HTTP 503 - temporary unavailability + "Throttling", // HTTP 400 - throttling variant + "ThrottledException", // HTTP 400 - throttling variant + }}; + // S3 error responses have the form ....... + // Require the root to avoid matching unrelated XML with a element. + static std::regex s3ErrorCodeRegex("[^]*([^<]+)"); + std::smatch s3Match; + bool isS3XmlStatus = httpStatus == 400 || httpStatus == 403 || httpStatus == 500 || httpStatus == 503; + auto s3ErrorCode = + (isS3XmlStatus && errorSink && std::regex_search(errorSink->s, s3Match, s3ErrorCodeRegex)) + ? s3Match[1].str() + : ""; + + if (std::find(s3RetryableErrors.begin(), s3RetryableErrors.end(), s3ErrorCode) + != s3RetryableErrors.end()) { + debug("S3 error '%s', will retry", s3ErrorCode); + } else if ( + httpStatus == HttpStatus::NotFound || httpStatus == HttpStatus::Gone + || code == CURLE_FILE_COULDNT_READ_FILE) { // The file is definitely not there err = NotFound; - } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { - // Don't retry on authentication/authorization failures + } else if (httpStatus == HttpStatus::Unauthorized || httpStatus == HttpStatus::ProxyAuthRequired) { + err = Unauthorized; + } else if (httpStatus == HttpStatus::Forbidden) { + // Don't retry on authentication/authorization failures. + // Note: the only reason we treat this differently from 401/407 is S3 returns 403 if a file doesn't + // exist and the bucket is unlistable. err = Forbidden; - } else if (httpStatus == 429) { - // 429 means too many requests, so we retry (with a substantially longer delay) - retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { + } else if ( + httpStatus >= 400 && httpStatus < 500 && httpStatus != HttpStatus::RequestTimeout + && httpStatus != HttpStatus::TooManyRequests) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again err = Misc; - } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { + } else if ( + httpStatus == HttpStatus::NotImplemented || httpStatus == HttpStatus::HttpVersionNotSupported + || httpStatus == HttpStatus::NetworkAuthRequired) { // Let's treat most 5xx (server) errors as transient, except for a handful: // * 501 not implemented // * 505 http version not supported @@ -715,19 +898,20 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.uri) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -735,50 +919,88 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); - /* If this is a transient error, then maybe retry the - download after a while. If we're writing to a - sink, we can only retry if the server supports - ranged requests. */ - if (err == Transient && attempt < fileTransfer.settings.tries - && (!this->request.dataCallback || writtenToSink == 0 || (acceptRanges && !hasContentEncoding))) { - int ms = retryTimeMs - * std::pow( - 2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); - - if (writtenToSink) { - warn( - "%s; retrying from offset %d in %d ms (attempt %d/%d)", - exc.message(), - writtenToSink, - ms, - attempt, - fileTransfer.settings.tries); - } else { - warn( - "%s; retrying in %d ms (attempt %d/%d)", - exc.message(), - ms, - attempt, - fileTransfer.settings.tries); - } + maybeRetry(err, httpStatus, std::move(exc)); + } + } - errorSink.reset(); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - try { - fileTransfer.enqueueItem(ref{shared_from_this()}); - } catch (const nix::Error & e) { - // If enqueue fails (e.g., during shutdown), fail the transfer properly - // instead of letting the exception propagate, which would leave done=false - // and cause the destructor to attempt a second callback invocation - fail(std::move(exc)); - } - } else - fail(std::move(exc)); + /* If this is a transient error, then maybe retry the download + after a while. If we're writing to a sink, we can only retry + if the server supports ranged requests. */ + void maybeRetry(FileTransfer::Error err, long httpStatus, FileTransferError && exc) + { + // Resolve effective retry config (request overrides > global settings) + auto effAttempts = request.retryAttempts.value_or(fileTransfer.settings.tries); + auto effBaseMs = request.retryDelayMs.value_or(fileTransfer.settings.retryDelayMs); + auto effRateLimitMs = + request.retryDelayRateLimitedMs.value_or(fileTransfer.settings.retryDelayRateLimitedMs); + auto effMaxMs = request.retryMaxDelayMs.value_or(fileTransfer.settings.retryMaxDelayMs); + + // Pick base delay by error class: 429/503 indicate the server is + // rate-limiting or overloaded, so use the longer rate-limit delay. + uint32_t baseMs = + (httpStatus == HttpStatus::TooManyRequests || httpStatus == HttpStatus::ServiceUnavailable) + ? effRateLimitMs + : effBaseMs; + + auto canRetry = [&] { + if (err != Transient) + return false; + if (attempt >= effAttempts) + return false; + // If we've already streamed bytes to the callback, we can + // resume via a Range request (if the server accepts byte + // ranges), or start over and discard the data we've already + // received. Neither works if the response is compressed (the + // Range applies to the encoded stream, but the sink saw + // decoded bytes). + if (request.dataCallback && writtenToSink != 0) + return !hasContentEncoding; + return true; + }(); + + if (!canRetry) { + fail(std::move(exc)); + return; + } + + auto delay = computeRetryDelayMs( + { + .attempt = attempt, + .baseMs = baseMs, + .ceilMs = effMaxMs, + .retryAfterMs = std::exchange(retryAfterMs, std::nullopt), + .jitter = fileTransfer.settings.retryJitter, + }, + fileTransfer.mt19937); + + if (writtenToSink) { + if (acceptRanges) + requestRange = true; + warn( + "%s; retrying from offset %d in %d ms (attempt %d/%d)", + exc.message(), + writtenToSink, + delay.count(), + attempt, + effAttempts); + } else { + warn("%s; retrying in %d ms (attempt %d/%d)", exc.message(), delay.count(), attempt, effAttempts); + } + + errorSink.reset(); + embargo = std::chrono::steady_clock::now() + delay; + try { + fileTransfer.enqueueItem(ref{shared_from_this()}); + } catch (const nix::Error & e) { + // If enqueue fails (e.g., during shutdown), fail the transfer properly + // instead of letting the exception propagate, which would leave done=false + // and cause the destructor to attempt a second callback invocation + fail(std::move(exc)); } } }; @@ -798,6 +1020,8 @@ struct curlFileTransfer : public FileTransfer private: bool quitting = false; public: + bool work = false; + void quit() { quitting = true; @@ -840,33 +1064,32 @@ struct curlFileTransfer : public FileTransfer workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~curlFileTransfer() - { - try { - stopWorkerThread(); - } catch (...) { - ignoreExceptionInDestructor(); - } - workerThread.join(); - } + ~curlFileTransfer() override; void stopWorkerThread() { /* Signal the worker thread to exit. */ - state_.lock()->quit(); - wakeupMulti(); + auto state(state_.lock()); + state->quit(); + wakeupMulti(*state); } - void wakeupMulti() + void wakeupMulti(State & state) { + state.work = true; if (auto ec = ::curl_multi_wakeup(curlm.get())) throw curlMultiError(ec); } void workerThreadMain() { + /* NOTE(cole-h): the maxQueueSize needs to be >0 or else things will hang */ + assert(maxQueueSize > 0); + /* Cause this thread to be notified on SIGINT. */ -#ifndef _WIN32 // TODO need graceful async exit support on Windows? +#if !defined(_WIN32) && !defined(IS_STATIC) // TODO need graceful async exit support on Windows? + // FIXME(RossComputerGuy): this causes issues on static builds. + // In particular, it causes a segfault to happen at the end of the program running. auto callback = createInterruptCallback([&]() { stopWorkerThread(); }); #endif @@ -908,25 +1131,12 @@ struct curlFileTransfer : public FileTransfer } } - /* Wait for activity, including wakeup events. */ - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() - ? std::max( - 0, - (int) std::chrono::duration_cast( - nextWakeup - std::chrono::steady_clock::now()) - .count()) - : maxSleepTimeMs; - - int numfds = 0; - mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw curlMultiError(mc); - nextWakeup = std::chrono::steady_clock::time_point(); std::vector> incoming; + std::vector> unpause; auto now = std::chrono::steady_clock::now(); + bool haveWork; { auto state(state_.lock()); @@ -948,25 +1158,23 @@ struct curlFileTransfer : public FileTransfer break; } } + unpause = std::exchange(state->unpause, {}); quit = state->isQuitting(); + haveWork = std::exchange(state->work, false); } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; items[item->req] = item; } - /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ - auto unpause = [&]() { - auto state(state_.lock()); - auto res = state->unpause; - state->unpause.clear(); - return res; - }(); + if (quit) + break; + /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ for (auto & item : unpause) { /* The transfer might have completed (failed) between it getting enqueued for unpause and by the time the worker thread picked @@ -976,6 +1184,26 @@ struct curlFileTransfer : public FileTransfer continue; static_cast(*ptr).unpause(); } + + /* Wait for activity, including wakeup events. */ + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int) std::chrono::duration_cast( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + + /* Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 (8.21), + curl_multi_perform seems to swallow queued up events ¯\_(ツ)_/¯. */ + if (haveWork) + sleepTimeMs = 0; + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw curlMultiError(mc); } debug("download thread shutting down"); @@ -1004,7 +1232,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1012,29 +1240,46 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); item->enqueued = true; /* Now any exceptions should be reported via the callback. */ + wakeupMulti(*state); } - wakeupMulti(); return ItemHandle(item.get_ptr()); } - ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback callback) override + inline ref + makeTransferItem(const FileTransferRequest & request, Callback callback) { /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */ if (request.uri.scheme() == "s3") { auto modifiedRequest = request; modifiedRequest.setupForS3(); - return enqueueItem(make_ref(*this, std::move(modifiedRequest), std::move(callback))); + return make_ref(*this, std::move(modifiedRequest), std::move(callback)); + } else { + return make_ref(*this, request, std::move(callback)); } + } + + ItemHandle + enqueueFileTransfer(const FileTransferRequest & request, Callback callback) noexcept override + { + const auto item = makeTransferItem(request, std::move(callback)); - return enqueueItem(make_ref(*this, request, std::move(callback))); + try { + return enqueueItem(item); + } catch (const nix::BaseError &) { + // NOTE(cole-h): catches both nix::Error and nix::Interrupted -- enqueueItem calls + // writeFull which may throw nix::Interrupted, and the rest of enqueueItem may throw + // nix::Error + item->failEx(std::current_exception()); + return ItemHandle(item.get_ptr()); + } } void unpauseTransfer(std::weak_ptr item) { auto state(state_.lock()); state->unpause.push_back(std::move(item)); - wakeupMulti(); + wakeupMulti(*state); } void unpauseTransfer(ItemHandle handle) override @@ -1045,19 +1290,39 @@ struct curlFileTransfer : public FileTransfer } }; +curlFileTransfer::~curlFileTransfer() +{ + try { + stopWorkerThread(); + } catch (...) { + ignoreExceptionInDestructor(); + } + workerThread.join(); +} + ref makeCurlFileTransfer(const FileTransferSettings & settings = fileTransferSettings) { return make_ref(settings); } +static auto * const _fileTransfer = new Sync>; + ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer->lock()); - if (fileTransfer->state_.lock()->isQuitting()) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = makeCurlFileTransfer().get_ptr(); + + return ref(*fileTransfer); +} - return fileTransfer; +std::shared_ptr resetFileTransfer() +{ + auto fileTransfer(_fileTransfer->lock()); + std::shared_ptr prev; + fileTransfer->swap(prev); + return prev; } ref makeFileTransfer(const FileTransferSettings & settings) @@ -1065,6 +1330,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1095,7 +1374,7 @@ void FileTransferRequest::setupForS3() #endif } -std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) +std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) noexcept { auto promise = std::make_shared>(); enqueueFileTransfer(request, {[promise](std::future fut) { @@ -1141,7 +1420,7 @@ void FileTransfer::download( bool paused = false; std::exception_ptr exc; std::string data; - std::condition_variable avail, request; + std::condition_variable avail; }; auto _state = std::make_shared>(); @@ -1151,10 +1430,9 @@ void FileTransfer::download( Finally finally([&]() { auto state(_state->lock()); state->quit = true; - state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) @@ -1197,7 +1475,6 @@ void FileTransfer::download( state->exc = std::current_exception(); } state->avail.notify_one(); - state->request.notify_one(); }}); while (true) { @@ -1231,8 +1508,6 @@ void FileTransfer::download( chunk = std::move(state->data); /* Reset state->data after the move, since we check data.empty() */ state->data = ""; - - state->request.notify_one(); } /* Flush the data to the sink and wake up the download thread @@ -1243,6 +1518,8 @@ void FileTransfer::download( } } +void FileTransferError::anchor() {} + template FileTransferError::FileTransferError( FileTransfer::Error error, std::optional response, const Args &... args) diff --git a/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh new file mode 100644 index 000000000000..7ff6d1fb4615 --- /dev/null +++ b/src/libstore/freebsd/build/chroot-freebsd-derivation-builder.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "chroot-derivation-builder.hh" +#include "freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" + +namespace nix { + +struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder +{ + std::shared_ptr autoDelJail = std::make_shared(); + + ChrootFreeBSDDerivationBuilder( + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params) + : DerivationBuilderImpl{store, miscMethods, params} + , ChrootDerivationBuilder{store, miscMethods, params} + , FreeBSDDerivationBuilder{store, miscMethods, params} + { + } + + virtual void cleanupBuild(bool force) override; + + void prepareSandbox() override; + + void startChild() override; + + void enterChroot() override; + + void addDependencyImpl(const StorePath & path) override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.cc b/src/libstore/freebsd/build/freebsd-derivation-builder.cc new file mode 100644 index 000000000000..68ccdafa1ae5 --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.cc @@ -0,0 +1,484 @@ +#include "derivation-builder-impl.hh" +#include "freebsd-derivation-builder.hh" +#include "chroot-derivation-builder.hh" +#include "chroot-freebsd-derivation-builder.hh" + +#include "nix/util/freebsd-jail.hh" +#include "nix/util/util.hh" +#include "nix/store/filetransfer.hh" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +FreeBSDDerivationBuilder::~FreeBSDDerivationBuilder() {} + +namespace { + +struct PasswordEntry +{ + std::string name; + uid_t uid; + gid_t gid; + std::string description; + std::filesystem::path home; + std::filesystem::path shell; +}; + +using UniqueDB = std::unique_ptr<::DB, decltype([](::DB * db) { + if (db) + (db->close)(db); + })>; + +// Database open flags from FreeBSD, in case they're necessary for compatibility +static constexpr HASHINFO dbFlags = { + .bsize = 4096, + .ffactor = 32, + .nelem = 256, + .cachesize = 2 * 1024 * 1024, + .hash = nullptr, + .lorder = BIG_ENDIAN, +}; + +// Password database version +// Version 4 has been current since 2003 +static const uint8_t dbVersion = 4; + +static void serializeString(std::vector & buf, std::string const & str) +{ + buf.insert(buf.end(), str.begin(), str.end()); + buf.push_back(0); +} + +static void serializeInt(std::vector & buf, uint32_t num) +{ + // Always big endian + buf.push_back((num >> 24) & 0xff); + buf.push_back((num >> 16) & 0xff); + buf.push_back((num >> 8) & 0xff); + buf.push_back((num >> 0) & 0xff); +} + +static std::vector byNameKey(std::string const & name) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; + buf.reserve(1 + name.size()); + // We can't use serializeString since that's null terminated + buf.insert(buf.end(), name.begin(), name.end()); + + return buf; +} + +static std::vector byNumKey(uint32_t num) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNUM, dbVersion)}; + serializeInt(buf, num); + + return buf; +} + +static std::vector byUidKey(uid_t uid) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYUID, dbVersion)}; + serializeInt(buf, uid); + + return buf; +} + +static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) +{ + auto db = + UniqueDB(::dbopen((chrootRootDir / "etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &dbFlags)); + + if (!db) + throw SysError("could not create password database"); + + auto dbInsert = [&db](std::vector keyBuf, std::vector & valueBuf) { + DBT key = {keyBuf.data(), keyBuf.size()}; + DBT value = {valueBuf.data(), valueBuf.size()}; + + if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { + throw SysError("could not write to password database"); + } + }; + + // Annoyingly DBT doesn't have const pointers so we need this whole shuffle + std::string versionKeyStr(_PWD_VERSION_KEY); + std::vector versionKey(versionKeyStr.begin(), versionKeyStr.end()); + std::vector versionValue{dbVersion}; + dbInsert(versionKey, versionValue); + + for (const auto & [i, user] : enumerate(users)) { + // flags for non-empty fields + uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; + + std::vector buf; + serializeString(buf, user.name); + // pw_password is always "*" in the insecure database + serializeString(buf, std::string("*")); + serializeInt(buf, user.uid); + serializeInt(buf, user.gid); + // pw_change = 0 means no requirement to change password + serializeInt(buf, 0); + // pw_class is empty since we don't make a class database + serializeString(buf, std::string("")); + serializeString(buf, user.description); + serializeString(buf, user.home); + serializeString(buf, user.shell); + // pw_expire = 0 means password does not expire + serializeInt(buf, 0); + serializeInt(buf, fields); + + dbInsert(byNameKey(user.name), buf); + // _PW_KEYBYNUM is 1-indexed + dbInsert(byNumKey(i + 1), buf); + dbInsert(byUidKey(user.uid), buf); + } + + // FreeBSD libc doesn't use /etc/passwd, but some software might + std::string passwdContent; + for (const auto & user : users) { + passwdContent.append( + fmt("%s:*:%d:%d:%s:%s:%s\n", + user.name, + user.uid, + user.gid, + user.description, + user.home.native(), + user.shell.native())); + } + + writeFile(chrootRootDir / "etc/passwd", passwdContent); + + // No need to make /etc/master.passwd or /etc/spwd.db, + // our build user wouldn't be able to read them anyway +} + +} // namespace + +template +struct iovec iovFromMutableBuffer(std::array & array) +{ + return { + .iov_base = static_cast(array.data()), + .iov_len = N, + }; +} + +template +struct iovec iovFromStaticSizedString(const char (&array)[N]) +{ + return { + .iov_base = const_cast(static_cast(array)), + .iov_len = N, + }; +} + +struct iovec iovFromDynamicSizeString(const std::string & s) +{ + return { + .iov_base = const_cast(static_cast(s.c_str())), + .iov_len = s.length() + 1, + }; +} + +void ChrootFreeBSDDerivationBuilder::cleanupBuild(bool force) +{ + autoDelJail->remove(); + ChrootDerivationBuilder::cleanupBuild(force); +} + +void ChrootFreeBSDDerivationBuilder::prepareSandbox() +{ + ChrootDerivationBuilder::prepareSandbox(); + + std::vector users{ + { + .name = "root", + .uid = 0, + .gid = 0, + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nixbld", + .uid = buildUser->getUID(), + .gid = sandboxGid(), + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nobody", + .uid = 65534, + .gid = 65534, + .description = "Nobody", + .home = "/", + .shell = "/noshell", + }, + }; + + createPasswordFiles(chrootRootDir, users); + + // FreeBSD doesn't have a group database, just write a text file + writeFile( + chrootRootDir / "etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); + + // Linux waits until after entering the child to start mounting so it doesn't + // pollute the root mount namespace. + // FreeBSD doesn't have mount namespaces, so there's no reason to wait. + + auto devpath = chrootRootDir / "dev"; + createDir(devpath, 0555); + createDir(chrootRootDir / "bin", 0555); + + std::array errmsg{}; + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("devfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(devpath.native()), + iovFromStaticSizedString("ruleset"), + iovFromStaticSizedString("4"), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount jail /dev: %1%", std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(devpath); + + for (const auto & [target, chrootPath] : pathsInChroot) { + std::filesystem::path path = chrootRootDir / target.relative_path(); + + auto maybeSt = maybeLstat(chrootPath.source); + if (!maybeSt) { + if (chrootPath.optional) + continue; /* Skip mounting this path. */ + else + throw SysError("getting attributes of path %1%", PathFmt(chrootPath.source)); + } + + /* Mount points must exist and be the right type. */ + if (S_ISDIR(maybeSt->st_mode)) { + createDirs(path); + } else if (S_ISLNK(maybeSt->st_mode)) { + createDirs(path.parent_path()); + copyFile(chrootPath.source, path, /*andDelete=*/false, /*contents=*/false); + continue; + } else { + createDirs(path.parent_path()); + writeFile(path, ""); + } + + std::array<::iovec, 8> iov = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("nullfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(path.native()), + iovFromStaticSizedString("target"), + iovFromDynamicSizeString(chrootPath.source.native()), + iovFromStaticSizedString("errmsg"), + iovFromMutableBuffer(errmsg), + }; + + debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); + + int flags = 0; + if (store.isInStore(target.native())) + /* While we are at it, enforce invariants about store paths. Anything located at the "logical" store + location must be readonly (file permission canonicalisation enforces this on the host filesystem). + Also the store must never contain setuid binaries for the same reason. This is just defense-in-depth. */ + flags = MNT_RDONLY | MNT_NOSUID; + + if (nmount(iov.data(), iov.size(), flags) < 0) + throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); + + autoDelJail->childrenMounts.emplace_back(path); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType.isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir / "etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (std::filesystem::path path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"}) { + if (pathExists(path)) { + // This means if your network config changes during a FOD build, + // the DNS in the sandbox will be wrong. However, this is pretty unlikely + // to actually be a problem, because FODs are generally pretty fast, + // and machines with often-changing network configurations probably + // want to run resolved or some other local resolver anyway. + // + // There's also just no simple way to do this correctly, you have to manually + // inotify watch the files for changes on the outside and update the sandbox + // while the build is running (or at least that's what Flatpak does). + // + // I also just generally feel icky about modifying sandbox state under a build, + // even though it really shouldn't be a big deal. -K900 + copyFile(path, chrootRootDir / path.relative_path(), false, true); + } + } + + if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir / "etc/ssl/certs"); + copyFile( + fileTransferSettings.caFile.get().value(), + chrootRootDir / "etc/ssl/certs/ca-certificates.crt", + false, + true); + } + } +} + +void ChrootFreeBSDDerivationBuilder::startChild() +{ + int jid; + + RunChildArgs args{ +#if NIX_WITH_AWS_AUTH + .awsCredentials = preResolveAwsCredentials(), +#endif + }; + + if (derivationType.isSandboxed()) { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + // TODO: Make our own ruleset + "vnet", + "new", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (isolated network): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + + // Everything from here to the end of the block is setting up the network + // code adapted from freebsd/sbin/ifconfig/af_inet.c, in_exec_nl + Pid helper = startProcess([&]() { + unix::closeExtraFDs(); + enterChroot(); + + struct snl_state ss = {}; + if (!snl_init(&ss, NETLINK_ROUTE)) { + throw SysError("Failed to init netlink connection"); + } + + struct snl_writer nw = {}; + snl_init_writer(&ss, &nw); + struct nlmsghdr * hdr = snl_create_msg_request(&nw, NL_RTM_NEWADDR); + struct ifaddrmsg * ifahdr = snl_reserve_msg_object(&nw, struct ifaddrmsg); + + ifahdr->ifa_family = AF_INET; + ifahdr->ifa_prefixlen = 8; + ifahdr->ifa_index = if_nametoindex("lo0"); + snl_add_msg_attr_ip4(&nw, IFA_LOCAL, (const struct in_addr *) "\x7f\x00\x00\x01"); + + int off = snl_add_msg_attr_nested(&nw, IFA_FREEBSD); + snl_add_msg_attr_u32(&nw, IFAF_FLAGS, IFF_LOOPBACK | IFF_UP); + snl_end_attr_nested(&nw, off); + + if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { + snl_free(&ss); + throw SysError("failed to sendoff netlink message"); + } + + struct snl_errmsg_data e = {}; + snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); + if (e.error_str != nullptr) { + snl_free(&ss); + throw SysError("failed to configure loopback interface: %1%", e.error_str); + } + snl_free(&ss); + _exit(0); + }); + + /* TODO: Capture the error from the helper? */ + if (auto status = helper.wait(); !statusOk(status)) { + throw Error("failed to configure loopback address: %s", statusToString(status)); + } + } else { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + // 4 is the most restrictive devfs ruleset that meets our needs + // which is found in the default installation. Trying to add + // another one is a huge pain... + "devfs_ruleset", + "4", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + "ip4", + "inherit", + "ip6", + "inherit", + "allow.raw_sockets", + "true", + nullptr); + if (jid < 0) { + throw SysError("failed to create jail (networked): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + } + + pid = startProcess([&]() { + openSlave(); + runChild(args); + }); +} + +void ChrootFreeBSDDerivationBuilder::enterChroot() +{ + /* Close all other file descriptors. This must happen before + jail_attach for FreeBSD. */ + unix::closeExtraFDs(); + + if (jail_attach(autoDelJail->jid) < 0) { + throw SysError("failed to attach to jail"); + } +} + +void ChrootFreeBSDDerivationBuilder::addDependencyImpl(const StorePath & path) +{ + throw UnimplementedError( + "adding store path '%s' to the sandbox is not implemented (recursive-nix)", store.printStorePath(path)); +} + +} // namespace nix diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.hh b/src/libstore/freebsd/build/freebsd-derivation-builder.hh new file mode 100644 index 000000000000..ce607db3982c --- /dev/null +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "derivation-builder-impl.hh" + +namespace nix { + +struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl +{ + using DerivationBuilderImpl::DerivationBuilderImpl; + + FreeBSDDerivationBuilder(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder(const FreeBSDDerivationBuilder &) = delete; + FreeBSDDerivationBuilder & operator=(FreeBSDDerivationBuilder &&) = delete; + FreeBSDDerivationBuilder & operator=(const FreeBSDDerivationBuilder &) = delete; + /* To appease Wweak-vtables. */ + ~FreeBSDDerivationBuilder() override; +}; + +} // namespace nix diff --git a/src/libstore/freebsd/meson.build b/src/libstore/freebsd/meson.build new file mode 100644 index 000000000000..bac3949b965f --- /dev/null +++ b/src/libstore/freebsd/meson.build @@ -0,0 +1,3 @@ +include_dirs += [ include_directories('build') ] + +sources += files('build/freebsd-derivation-builder.cc') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 16b81abf2821..73adbac343a1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,9 +1,10 @@ -#include "nix/store/derivations.hh" -#include "nix/store/globals.hh" +#include "nix/store/gc-store.hh" #include "nix/store/local-gc.hh" +#include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" #include "nix/util/configuration.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" @@ -22,6 +23,7 @@ #include #include #include +#include #if HAVE_STATVFS # include #endif @@ -35,6 +37,10 @@ namespace nix { +void LocalSettings::anchor() {} + +void GCSettings::anchor() {} + static std::string gcSocketPath = "gc-socket/socket"; static std::string gcRootsDir = "gcroots"; @@ -75,7 +81,7 @@ void LocalStore::createTempRootsFile() } } -void LocalStore::addTempRoot(const StorePath & path) +void LocalStore::addTempRoots(const StorePathSet & paths) { if (config->readOnly) { debug( @@ -124,12 +130,14 @@ void LocalStore::addTempRoot(const StorePath & path) } try { - debug("sending GC root '%s'", printStorePath(path)); - writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); - char c; - readFull(fdRootsSocket->get(), &c, 1); - assert(c == '1'); - debug("got ack for GC root '%s'", printStorePath(path)); + for (auto & path : paths) { + debug("sending GC root '%s'", printStorePath(path)); + writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); + char c; + readFull(fdRootsSocket->get(), &c, 1); + assert(c == '1'); + debug("got ack for GC root '%s'", printStorePath(path)); + } } catch (SystemError & e) { /* The garbage collector may have exited, so we need to restart. */ @@ -146,10 +154,22 @@ void LocalStore::addTempRoot(const StorePath & path) } } - /* Record the store path in the temporary roots file so it will be + /* Record the store paths in the temporary roots file so they will be seen by a future run of the garbage collector. */ - auto s = printStorePath(path) + '\0'; - writeFull(_fdTempRoots.lock()->get(), s); + + std::string s; + + for (auto & path : paths) + s += printStorePath(path) + '\0'; + + { + auto fdTempRoots(_fdTempRoots.lock()); + + /* This might not be atomic, but that's fine. Writes go in-order, and if + we partially write a store path, findTempRoots() will just ignore it, + and we'll send it the new temproots below if it's still running. */ + writeFull(fdTempRoots->get(), s); + } } static std::string censored = "{censored}"; @@ -168,8 +188,6 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) } auto path = i.path(); - pid_t pid = std::stoi(name); - debug("reading temporary root file %1%", PathFmt(path)); AutoCloseFD fd(toDescriptor(open( path.string().c_str(), @@ -204,7 +222,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != std::string::npos) { auto root = std::string_view(contents).substr(pos, end - pos); debug("got temporary root '%s'", root); - tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%s}", name)); pos = end + 1; } } @@ -355,16 +373,23 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; - boost::unordered_flat_set> roots, dead, alive; + Roots roots; + boost::unordered_flat_set> dead, alive; + + /* Return early if nothing to delete */ + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) + return; struct Shared { // The temp roots only store the hash part to make it easier to // ignore suffixes like '.lock', '.chroot' and '.check'. - boost::unordered_flat_set> tempRoots; + boost::unordered_flat_map tempRoots; // Hash part of the store path currently being deleted, if // any. @@ -375,15 +400,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -476,7 +492,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) debug("got new GC root '%s'", path); auto hashPart = storePath->hashPart(); auto shared(_shared.lock()); - shared->tempRoots.emplace(hashPart); + // FIXME: could get the PID from the socket. + shared->tempRoots.insert_or_assign(std::string(hashPart), "{nix-process:unknown}"); /* If this path is currently being deleted, then we have to wait until deletion is finished to ensure that @@ -516,20 +533,16 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); - Roots rootMap; if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); - - for (auto & i : rootMap) - roots.insert(i.first); + findRootsNoTemp(roots, options.censor); /* Read the temporary roots created before we acquired the global GC root. Any new roots will be sent to our socket. */ - Roots tempRoots; - findTempRoots(tempRoots, true); - for (auto & root : tempRoots) { - _shared.lock()->tempRoots.emplace(root.first.hashPart()); - roots.insert(root.first); + { + Roots tempRoots; + findTempRoots(tempRoots, options.censor); + for (auto & root : tempRoots) + _shared.lock()->tempRoots.insert_or_assign(std::string(root.first.hashPart()), *root.second.begin()); } /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */ @@ -548,7 +561,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - auto tmpDirFd = openDirectory(realPath); + /* TODO Reconsider whether Follow is the right choice, here */ + auto tmpDirFd = openDirectory(realPath, FinalSymlink::Follow); if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir %s", PathFmt(realPath)); return; @@ -559,6 +573,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); + /* If this path has a leftover `.unpacked` marker (from an + interrupted `addMultipleToStore()`), delete the marker + *before* the path itself. Deleting a directory is not atomic, + so if we were interrupted partway through, a marker left next + to a partially-deleted path would make `addMultipleToStore()` + reuse that corrupt path. */ + deletePath(unpackedMarkerFor(realPath)); + uint64_t bytesFreed; deleteStorePath(realPath, bytesFreed, isKnownPath); @@ -576,7 +598,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) via the referrers edges and optionally derivers and derivation output edges. If none of those paths are roots, then all visited paths are garbage and are deleted. */ - auto deleteReferrersClosure = [&](const StorePath & start) { + auto maybeDeleteReferrersClosure = [&](const StorePath & start) { StorePathSet visited; std::queue todo; @@ -593,6 +615,33 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) todo.push(path); }; + auto markAlive = [&](const StorePath & p) { + alive.insert(p); + try { + StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); + computeFSClosure( + p, + closure, + /* flipDirection */ false, + includeOutputs, + includeDerivers); + for (auto & c : closure) + alive.insert(c); + } catch (InvalidPath &) { + } + }; + enqueue(start); while (auto path = pop(todo)) { @@ -600,48 +649,53 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; - auto markAlive = [&]() { - alive.insert(*path); - alive.insert(start); - try { - StorePathSet closure; - computeFSClosure( - *path, - closure, - /* flipDirection */ false, - keepOutputs, - keepDerivations); - for (auto & p : closure) - alive.insert(p); - } catch (InvalidPath &) { + if (auto pathsToDelete = std::get_if(&options.pathsToDelete)) { + if (!pathsToDelete->deleteReferrers && !pathsToDelete->paths.contains(*path)) { + if (options.action != GCOptions::gcDeleteDead) + throw Error( + "Cannot delete path '%s' because it's referenced by path '%s'.", + printStorePath(start), + printStorePath(*path)); + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return; } - }; - + } /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (auto i = roots.find(*path); i != roots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's referenced by the GC root '%s'.", + printStorePath(start), + *i->second.begin()); debug("cannot delete '%s' because it's a root", printStorePath(*path)); - return markAlive(); + alive.insert(start); + return markAlive(*path); } - if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) - return; - - { + static bool inTest = getEnv("_NIX_IN_TEST").has_value(); + if (!(inTest && options.ignoreLiveness)) { auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { - debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); - return markAlive(); + if (auto i = shared->tempRoots.find(std::string(hashPart)); i != shared->tempRoots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's in use by '%s'.", printStorePath(start), i->second); + alive.insert(start); + return markAlive(*path); } shared->pending = hashPart; } @@ -659,27 +713,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, then visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const GCOptions::SpecificPaths &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { if (!dead.insert(path).second) continue; if (shouldDelete) { + /* Re-check tempRoots before deleting and set pending + to synchronise with addTempRoot. Between the BFS + and this deletion loop, new temproots may have been + added via the GC socket by a concurrent process + (e.g. an evaluator calling addTempRoot). The BFS + only checks tempRoots when it first visits a path, + but the "pending" mechanism only blocks the socket + handler for the single path currently being visited, + not for paths already queued for deletion. */ + { + auto hashPart = std::string(path.hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.contains(hashPart)) { + debug( + "not deleting '%s' because it became a temporary root after initial scan", + printStorePath(path)); + markAlive(path); + continue; + } + shared->pending = hashPart; + } + Finally resetPending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); try { invalidatePathChecked(path); deleteFromStore(path.to_string(), true); @@ -693,50 +780,67 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { - - for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); - } - - } else if (options.maxFreed > 0) { + try { + /* Either delete all garbage paths, or just the specified paths. */ + std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (shouldDelete) - printInfo("deleting garbage..."); - else - printInfo("determining live/dead paths..."); + for (auto & i : pathsToDelete.paths) { + maybeDeleteReferrersClosure(i); + assert(options.action == GCOptions::gcDeleteDead || dead.count(i)); + } + }, + [&](const GCOptions::WholeStore & _) { + if (options.maxFreed == 0) + return; - try { - AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); - if (!dir) - throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); - - /* Read the store and delete all paths that are invalid or - unreachable. We don't use readDirectory() here so that - GCing can start faster. */ - auto linksName = linksDir.filename(); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == ".." || name == linksName) - continue; + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage..."); + break; + case GCOptions::gcDeleteSpecific: + throw Error("Cannot delete the entire store"); + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) - deleteReferrersClosure(*storePath); - else - deleteFromStore(name, false); - } - } catch (GCLimitReached & e) { - } + AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); + if (!dir) + throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); + + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = linksDir.filename(); + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == ".." || name == linksName) + continue; + + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + maybeDeleteReferrersClosure(*storePath); + else + deleteFromStore(name, false); + } + }, + }, + options.pathsToDelete); + } catch (GCLimitReached & e) { } if (options.action == GCOptions::gcReturnLive) { @@ -851,12 +955,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -883,7 +992,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2e262aec8a0a..57c17680f783 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,12 +4,10 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" -#include "nix/util/archive.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" #include "nix/util/executable-path.hh" -#include "nix/util/signals.hh" #include "nix/store/filetransfer.hh" #include @@ -20,6 +18,8 @@ #include #include +#include + #ifndef _WIN32 # include #endif @@ -34,8 +34,6 @@ # include "nix/util/processes.hh" #endif -#include "nix/util/config-impl.hh" - #ifdef __APPLE__ # include #endif @@ -48,46 +46,33 @@ namespace nix { -/* The default location of the daemon socket, relative to nixStateDir. - The socket is in a directory to allow you to control access to the - Nix daemon by setting the mode/ownership of the directory - appropriately. (This wouldn't work on the socket itself since it - must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "daemon-socket/socket" +void Settings::anchor() {} -/** - * Helper to resolve the NIX_CONF_DIR at runtime on Windows. - * On Windows, NIX_CONF_DIR is not defined at compile time, so we determine - * the path at runtime using the Windows known folders API (FOLDERID_ProgramData). - * This allows Nix to work correctly regardless of which drive Windows is installed on. - */ -static std::filesystem::path resolveNixConfDir() -{ -#ifdef _WIN32 -# ifdef NIX_CONF_DIR - // On Windows, NIX_CONF_DIR should not be defined at compile time -# error "NIX_CONF_DIR should not be defined on Windows" -# endif - return windows::known_folders::getProgramData() / "nix"; -#else - return NIX_CONF_DIR; -#endif -} +void NarInfoDiskCacheSettings::anchor() {} -LogFileSettings::LogFileSettings() - : nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) -{ -} +void LogFileSettings::anchor() {} + +void AutoAllocateUidSettings::anchor() {} Settings settings; static GlobalConfig::Register rSettings(&settings); Settings::Settings() - : nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixDaemonSocketFile(canonPath(getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(nixStateDir / DEFAULT_SOCKET_PATH))) + : nixStateDir(getEnvOsNonEmpty(OS_STR("NIX_STATE_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_STATE_DIR +# error "NIX_STATE_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "state"; +#else + return NIX_STATE_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { #ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; @@ -151,12 +136,28 @@ void loadConfFile(AbstractConfig & config) } } +/** + * On Windows, NIX_CONF_DIR (and other directories like NIX_STATE_DIR, NIX_LOG_DIR) + * are not defined at compile time, so we determine paths at runtime using the + * Windows known folders API (FOLDERID_ProgramData). This allows Nix to work + * correctly regardless of which drive Windows is installed on. + */ const std::filesystem::path & nixConfDir() { - static const std::filesystem::path dir = - canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(resolveNixConfDir())); + static const std::filesystem::path dir = getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_CONF_DIR +# error "NIX_CONF_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "conf"; +#else + return NIX_CONF_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value(); return dir; } @@ -288,6 +289,20 @@ const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported( return nullptr; } +std::optional WorkerSettings::getHostName() +{ + if (hostName != "") + return hostName; + +#ifndef _WIN32 + char hostname[_POSIX_HOST_NAME_MAX + 1]; + if (gethostname(hostname, sizeof(hostname)) == 0) + return std::string(hostname); +#endif + + return std::nullopt; +} + ProfileDirsOptions Settings::getProfileDirsOptions() const { return { @@ -298,6 +313,8 @@ ProfileDirsOptions Settings::getProfileDirsOptions() const std::string nixVersion = PACKAGE_VERSION; +const std::string determinateNixVersion = DETERMINATE_NIX_VERSION; + NLOHMANN_JSON_SERIALIZE_ENUM( SandboxMode, { @@ -375,6 +392,27 @@ void from_json(const nlohmann::json & j, ChrootPath & cp) cp.optional = j.at("optional").get(); } +static nlohmann::json pathsInChrootToJSON(const PathsInChroot & paths) +{ + auto j = nlohmann::json::object(); + for (auto & [target, chrootPath] : paths) { + nlohmann::json cp; + to_json(cp, chrootPath); + j[target.string()] = std::move(cp); + } + return j; +} + +template<> +std::map BaseSetting::toJSONObject() const +{ + auto obj = AbstractSetting::toJSONObject(); + obj.emplace("value", pathsInChrootToJSON(value)); + obj.emplace("defaultValue", pathsInChrootToJSON(defaultValue)); + obj.emplace("documentDefault", documentDefault); + return obj; +} + template<> PathsInChroot BaseSetting::parse(const std::string & str) const { @@ -444,6 +482,24 @@ std::string BaseSetting::to_string() const return nlohmann::json(value).dump(); } +template +T JSONSetting::parse(const std::string & str) const +{ + try { + return nlohmann::json::parse(str).template get(); + } catch (std::exception & e) { + throw UsageError("parsing setting '%s': %s", BaseSetting::name, e.what()); + } +} + +template +std::string JSONSetting::to_string() const +{ + return nlohmann::json(BaseSetting::get()).dump(); +} + +template class JSONSetting; + template<> void BaseSetting::appendOrSet(PathsInChroot newValue, bool append) { diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index b3678ae4fdf1..eb4613ecc24d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -4,13 +4,18 @@ #include "nix/store/nar-info-disk-cache.hh" #include "nix/store/sqlite.hh" #include "nix/util/callback.hh" +#include "nix/util/closure.hh" #include "nix/store/store-registration.hh" #include "nix/store/globals.hh" +#include "nix/util/compression.hh" +#include "nix/util/topo-sort.hh" namespace nix { MakeError(UploadToHTTP, Error); +void UploadToHTTP::anchor() {} + StringSet HttpBinaryCacheStoreConfig::uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; @@ -20,8 +25,12 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) , cacheUri(std::move(_cacheUri)) { @@ -67,18 +76,59 @@ void HttpBinaryCacheStore::init() auto cacheKey = config->getReference().render(/*withParams=*/false); if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + applyCacheInfoFields(cacheInfo->fields); } else { + std::map fields; try { - BinaryCacheStore::init(); + fields = parseNixCacheInfo(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } - diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); + applyCacheInfoFields(fields); + diskCache->createCache(cacheKey, config->storeDir, {.fields = std::move(fields)}); } } +StorePaths HttpBinaryCacheStore::topoSortPaths(const StorePathSet & paths) +{ + std::unordered_map> pathInfos; + StorePathSet referencesClosureSet; + + /* Traverse the references closure that is also present in the starting set + in an asynchronous manner. */ + computeClosure( + paths, + referencesClosureSet, + [this, &paths, &pathInfos](const StorePath & path) -> asio::awaitable { + StorePathSet res; + auto info = co_await callbackToAwaitable>( + [this, path](Callback> cb) { queryPathInfo(path, std::move(cb)); }); + + for (auto & ref : info->references) + /* Don't traverse into items that don't exist in our starting set. */ + if (ref != path && paths.count(ref)) + res.insert(ref); + + /* Fill the map. */ + pathInfos.emplace(path, info); + + co_return res; + }); + + auto result = topoSort(paths, [&](const StorePath & path) { return pathInfos.at(path)->references; }); + + return std::visit( + overloaded{ + [&](const Cycle & cycle) -> StorePaths { + throw Error( + "cycle detected in the references of '%s' from '%s'", + printStorePath(cycle.path), + printStorePath(cycle.parent)); + }, + [](const auto & sorted) { return sorted; }}, + result); +} + std::optional HttpBinaryCacheStore::getCompressionMethod(const std::string & path) { if (hasSuffix(path, ".narinfo") && config->narinfoCompression.get()) @@ -160,18 +210,18 @@ void HttpBinaryCacheStore::upsertFile( { try { if (auto compressionMethod = getCompressionMethod(path)) { - CompressedSource compressed(source, *compressionMethod); + StringSource compressed(compress(*compressionMethod, source)); /* TODO: Validate that this is a valid content encoding. We probably shouldn't set non-standard values here. */ Headers headers = {{"Content-Encoding", showCompressionAlgo(*compressionMethod)}}; - upload(path, compressed, compressed.size(), mimeType, std::move(headers)); + upload(path, compressed, compressed.s.size(), mimeType, std::move(headers)); } else { upload(path, source, sizeHint, mimeType, std::nullopt); } } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } @@ -212,6 +262,18 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) } } + // Propagate per-substituter retry overrides to the transfer request. + // Only set when the user actually specified the URL parameter; otherwise + // the transfer falls back to the global FileTransferSettings. + auto propagate = [](auto & setting, auto & dest) { + if (setting.isOverridden()) + dest = setting.get(); + }; + propagate(config->retryDelayMs, request.retryDelayMs); + propagate(config->retryDelayRateLimitedMs, request.retryDelayRateLimitedMs); + propagate(config->retryMaxDelayMs, request.retryMaxDelayMs); + propagate(config->retryAttempts, request.retryAttempts); + return request; } diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2caf..03dd350ec518 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/active-builds.hh b/src/libstore/include/nix/store/active-builds.hh new file mode 100644 index 000000000000..a56b33f5b5ff --- /dev/null +++ b/src/libstore/include/nix/store/active-builds.hh @@ -0,0 +1,116 @@ +#pragma once + +#include "nix/util/util.hh" +#include "nix/util/json-impls.hh" +#include "nix/store/path.hh" + +#include +#include + +namespace nix { + +/** + * A uid and optional corresponding user name. + */ +struct UserInfo +{ + uid_t uid = -1; + std::optional name; + + /** + * Create a UserInfo from a UID, looking up the username if possible. + */ + static UserInfo fromUid(uid_t uid); +}; + +struct ActiveBuild +{ + pid_t nixPid; + + std::optional clientPid; + std::optional clientUid; + + pid_t mainPid; + UserInfo mainUser; + std::optional cgroup; + + time_t startTime; + + StorePath derivation; +}; + +struct ActiveBuildInfo : ActiveBuild +{ + struct ProcessInfo + { + pid_t pid = 0; + pid_t parentPid = 0; + UserInfo user; + std::vector argv; + std::optional utime, stime, cutime, cstime; + }; + + // User/system CPU time for the entire cgroup, if available. + std::optional utime, stime; + + std::vector processes; +}; + +struct TrackActiveBuildsStore +{ + struct BuildHandle + { + TrackActiveBuildsStore & tracker; + uint64_t id; + + BuildHandle(TrackActiveBuildsStore & tracker, uint64_t id) + : tracker(tracker) + , id(id) + { + } + + BuildHandle(BuildHandle && other) noexcept + : tracker(other.tracker) + , id(other.id) + { + other.id = 0; + } + + ~BuildHandle() + { + if (id) { + try { + tracker.buildFinished(*this); + } catch (...) { + ignoreExceptionInDestructor(); + } + } + } + }; + + virtual ~TrackActiveBuildsStore() = default; + + virtual BuildHandle buildStarted(const ActiveBuild & build) = 0; + + virtual void buildFinished(const BuildHandle & handle) = 0; + + virtual void anchor(); +}; + +struct QueryActiveBuildsStore +{ + inline static std::string operationName = "Querying active builds"; + + virtual ~QueryActiveBuildsStore() = default; + + virtual std::vector queryActiveBuilds() = 0; + + virtual void anchor(); +}; + +} // namespace nix + +JSON_IMPL(UserInfo) +JSON_IMPL(ActiveBuild) +JSON_IMPL(ActiveBuildInfo) +JSON_IMPL(ActiveBuildInfo::ProcessInfo) diff --git a/src/libstore/include/nix/store/async-path-writer.hh b/src/libstore/include/nix/store/async-path-writer.hh new file mode 100644 index 000000000000..49d1ecbc69fd --- /dev/null +++ b/src/libstore/include/nix/store/async-path-writer.hh @@ -0,0 +1,27 @@ +#pragma once + +#include "nix/store/store-api.hh" + +namespace nix { + +struct AsyncPathWriter +{ + virtual void anchor(); + + virtual ~AsyncPathWriter() = default; + + virtual StorePath addPath( + std::string contents, + std::string name, + StorePathSet references, + RepairFlag repair, + std::shared_ptr provenance = {}) = 0; + + virtual void waitForPath(const StorePath & path) = 0; + + virtual void waitForAllPaths() = 0; + + static ref make(ref store); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 0751757cb015..3fa747069935 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -36,6 +36,10 @@ struct AwsCredentials class AwsAuthError final : public CloneableError { +private: + void anchor() override; + +public: std::optional errorCode; public: @@ -70,7 +74,7 @@ public: } } - virtual ~AwsCredentialProvider() {} + virtual ~AwsCredentialProvider(); }; /** diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 2c3fbdc3a0d1..1544c4bcec82 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,7 +16,14 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; +private: + void anchor() override; + +public: + BinaryCacheStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + { + } Setting compression{ this, @@ -39,8 +46,8 @@ struct BinaryCacheStoreConfig : virtual StoreConfig fetch debug info on demand )"}; - Setting secretKeyFile{ - this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + Setting> secretKeyFile{ + this, std::nullopt, "secret-key", "Path to the secret key used to sign the binary cache."}; Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; @@ -55,7 +62,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + R"( + Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`. + + If not set explicitly, defaults to `true` when `compression` is `zstd` and `false` otherwise. + )"}; Setting compressionLevel{ this, @@ -85,28 +96,55 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895). This is call "v2" accordingly. + * + * While we're experimenting, we'll freely increase this version + * number. Old build traces will just be "abandoned" at the old URL. + * When we are done experimenting, we'll try lean more on versioning + * the build trace entries themselves than the entire directory, for + * a smoother migration path. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace-v2"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; BinaryCacheStore(Config &); + /** + * Fetch and parse `nix-cache-info`. + */ + std::map parseNixCacheInfo(); + + /** + * Apply the known `nix-cache-info` fields from `fields` to this store. + */ + void applyCacheInfoFields(const std::map & fields); + /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); public: + bool includeInProvenance() override + { + return true; + } + virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile( @@ -160,6 +198,23 @@ private: void writeNarInfo(ref narInfo); + /** + * Upload the NAR for a path and everything else *except* the + * `.narinfo` file (i.e. the compressed NAR, an optional NAR + * listing, and optional debuginfo links), and construct the + * corresponding `NarInfo`. The returned `NarInfo` is neither signed + * nor published yet; call `uploadNarInfo()` to do that. + */ + ref uploadData(Source & narSource, RepairFlag repair, fun mkInfo); + + /** + * Sign and publish the `.narinfo` file for a path whose NAR has + * already been uploaded by `uploadData()`. This is what establishes + * the closure invariant, so all of the path's references must + * already be valid in the store. + */ + void uploadNarInfo(ref narInfo); + ref addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo); @@ -180,6 +235,9 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; + void + addMultipleToStore(PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) override; + StorePath addToStoreFromDump( Source & dump, std::string_view name, @@ -187,7 +245,8 @@ public: ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override; + RepairFlag repair, + std::shared_ptr provenance) override; StorePath addToStore( std::string_view name, diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index c664e6e5b6f7..c3bdb2b09973 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -13,6 +13,8 @@ namespace nix { +struct Provenance; + /** * Names must be disjoint with `BuildResultFailureStatus`. * @@ -50,6 +52,7 @@ enum struct BuildResultFailureStatus : uint8_t { /// know about this one, so change it back to `OutputRejected` /// before serialization. HashMismatch, + Cancelled, }; /** @@ -58,8 +61,11 @@ enum struct BuildResultFailureStatus : uint8_t { * This is both an exception type (inherits from Error) and serves as * the failure variant in BuildResult::inner. */ -struct BuildError : public CloneableError +class BuildError : public CloneableError { + void anchor() override; + +public: using Status = BuildResultFailureStatus; using enum Status; @@ -73,6 +79,11 @@ struct BuildError : public CloneableError */ bool isNonDeterministic = false; + /** + * The provenance of the derivation, if any. + */ + std::shared_ptr provenance; + public: /** * Variadic constructor for throwing with format strings. @@ -90,6 +101,7 @@ public: Status status; HintFmt msg; bool isNonDeterministic = false; + std::shared_ptr provenance; }; /** @@ -100,7 +112,7 @@ public: : CloneableError(std::move(args.msg)) , status{args.status} , isNonDeterministic{args.isNonDeterministic} - + , provenance{args.provenance} { } @@ -130,6 +142,12 @@ struct BuildResult */ SingleDrvOutputs builtOutputs; + /** + * The provenance of the derivation, if any. Note that this is the provenance of the current build, not + * necessarily of previously existing outputs. + */ + std::shared_ptr provenance; + bool operator==(const BuildResult::Success &) const noexcept; std::strong_ordering operator<=>(const BuildResult::Success &) const noexcept; }; @@ -192,6 +210,13 @@ struct BuildResult bool operator==(const BuildResult &) const noexcept; std::strong_ordering operator<=>(const BuildResult &) const noexcept; + + bool isCancelled() const + { + auto failure = tryGetFailure(); + // FIXME: remove MiscFailure eventually. + return failure && (failure->status == Failure::Cancelled || failure->status == Failure::MiscFailure); + } }; /** diff --git a/src/libstore/include/nix/store/build/build-log.hh b/src/libstore/include/nix/store/build/build-log.hh index cdc9125734d1..4fd71a65f5d2 100644 --- a/src/libstore/include/nix/store/build/build-log.hh +++ b/src/libstore/include/nix/store/build/build-log.hh @@ -3,6 +3,7 @@ #include "nix/util/logging.hh" #include "nix/util/serialise.hh" +#include "nix/util/ref.hh" #include #include @@ -35,9 +36,9 @@ private: public: /** - * The build activity. Owned by BuildLog. + * The build activity. */ - std::unique_ptr act; + ref act; /** * Map for tracking nested activities from JSON messages. @@ -48,7 +49,7 @@ public: * @param maxTailLines Maximum number of tail lines to keep * @param act Activity for this build */ - BuildLog(size_t maxTailLines, std::unique_ptr act); + BuildLog(size_t maxTailLines, ref act); /** * Process output data from child process. diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index f521c0402a2f..2562e29da743 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -16,12 +16,27 @@ namespace nix { +/** + * Rethrow the current exception as a subclass of `Error`. + */ +void rethrowExceptionAsError(); + +/** + * Send the current exception to the parent in the format expected by + * `DerivationBuilderImpl::processSandboxSetupMessages()`. + */ +void handleChildException(bool sendException); + /** * Denotes a build failure that stemmed from the builder exiting with a * failing exist status. */ struct BuilderFailureError final : CloneableError { +private: + void anchor() override; + +public: int builderStatus; std::string extraMsgAfter; @@ -48,6 +63,9 @@ struct ChrootPath bool optional = false; }; +void to_json(nlohmann::json & j, const ChrootPath & cp); +void from_json(const nlohmann::json & j, ChrootPath & cp); + typedef std::map PathsInChroot; // maps target path to source path /** @@ -58,6 +76,11 @@ struct DerivationBuilderParams /** The path of the derivation. */ const StorePath & drvPath; + /** + * The provenance of the derivation, if known + */ + const std::shared_ptr drvProvenance; + BuildResult & buildResult; /** @@ -99,6 +122,11 @@ struct DerivationBuilderParams StringSet systemFeatures; DesugaredEnv desugaredEnv; + + /** + * The activity corresponding to the build. + */ + ref act; }; /** @@ -106,7 +134,7 @@ struct DerivationBuilderParams */ struct DerivationBuilderCallbacks { - virtual ~DerivationBuilderCallbacks() = default; + virtual ~DerivationBuilderCallbacks(); /** * Open a log file and a pipe to it. @@ -137,6 +165,10 @@ struct DerivationBuilderCallbacks */ struct DerivationBuilder : RestrictionContext { +private: + void anchor() override; + +public: DerivationBuilder() = default; virtual ~DerivationBuilder() = default; @@ -212,7 +244,7 @@ using DerivationBuilderUnique = std::unique_ptr miscMethods, DerivationBuilderParams params); + LocalStore & store, std::shared_ptr miscMethods, DerivationBuilderParams params); /** * @param handler Must be chosen such that it supports the given @@ -220,7 +252,7 @@ DerivationBuilderUnique makeDerivationBuilder( */ DerivationBuilderUnique makeExternalDerivationBuilder( LocalStore & store, - std::unique_ptr miscMethods, + std::shared_ptr miscMethods, DerivationBuilderParams params, const ExternalBuilder & handler); #endif diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 826cf2c50f07..4d366f648c08 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -31,6 +31,8 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; */ struct DerivationBuildingGoal : public Goal { + friend class Worker; + /** * @param storeDerivation Whether to store the derivation in * `worker.store`. This is useful for newly-resolved derivations. In this @@ -41,7 +43,11 @@ struct DerivationBuildingGoal : public Goal * faithfully reconstruct the build history. */ DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, + ref drv, + Worker & worker, + BuildMode buildMode, + bool storeDerivation); ~DerivationBuildingGoal(); private: @@ -52,7 +58,7 @@ private: /** * The derivation stored at drvPath. */ - const std::unique_ptr drv; + const ref drv; /** * The remainder is state held during the build. @@ -109,7 +115,10 @@ private: */ std::pair checkPathValidity(std::map & initialOutputs); - Done doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs); + Done doneSuccess( + BuildResult::Success::Status status, + SingleDrvOutputs builtOutputs, + std::shared_ptr provenance = nullptr); Done doneFailure(BuildError ex); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa1782a4..8d6892839c76 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index aaded75511f0..94aba5c3160d 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -45,7 +45,7 @@ struct DerivationGoal : public Goal */ DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -64,9 +64,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; - - const Hash outputHash; + ref drv; const BuildMode buildMode; diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index b79e6bbb79e4..972558e9706f 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -35,7 +35,10 @@ struct BuilderFailureError; */ struct DerivationResolutionGoal : public Goal { - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + friend class Worker; + + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -53,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 5f36bbf06d8b..6652da6a20c8 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -20,11 +20,12 @@ class Worker; * If the output store object itself should also be substituted, that is * the responsibility of the caller to do so. * - * @todo rename this `BuidlTraceEntryGoal`, which will make sense + * @todo rename this `BuildTraceEntryGoal`, which will make sense * especially once `Realisation` is renamed to `BuildTraceEntry`. */ class DrvOutputSubstitutionGoal : public Goal { + friend class Worker; /** * The drv output we're trying to substitute diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 263647fc126a..121d2b058d63 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -10,8 +10,11 @@ namespace nix { -struct TimedOut final : CloneableError +class TimedOut final : public CloneableError { + void anchor() override; + +public: time_t maxDuration; TimedOut(time_t maxDuration); @@ -75,11 +78,59 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); +public: + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + +private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ Goals waitees; + /** + * Memoised result of key(). + */ + std::optional cachedKey; + + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -118,7 +169,7 @@ public: /** * Build result. */ - BuildResult buildResult; + BuildResult buildResult{BuildError(BuildResult::Failure::Cancelled, "")}; /** * Suspend our goal and wait until we get `work`-ed again. @@ -147,22 +198,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -228,8 +263,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() @@ -253,6 +290,12 @@ public: void await_resume() {}; }; + template + struct AsyncCallback + { + fun)> fn; + }; + /** * Used on initial suspend, does the same as `std::suspend_always`, * but asserts that everything has been set correctly. @@ -308,28 +351,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -423,6 +444,9 @@ public: return static_cast(co); } + template + auto await_transform(AsyncCallback && acb); + /** * Awaiter for @ref Suspend. Always suspends, but asserts * there are no pending child events (those should be @@ -434,7 +458,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -462,7 +486,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -473,7 +497,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; @@ -591,6 +615,17 @@ public: */ virtual std::string key() = 0; + /** + * Memoising variant of key(). We really don't want to pay the overhead of + * allocating strings just to compare Goals. + */ + std::string_view keyCached() & + { + if (cachedKey) + return *cachedKey; + return *(cachedKey = key()); + } + /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory @@ -600,7 +635,19 @@ public: protected: Co await(Goals waitees); + /** + * Awaiting on the resulting coroutine yields the goal for several seconds. + * Used for retrying goals blocked on acquiring lockfiles. + */ Co waitForAWhile(); + + /** + * Awaiting on the resulting coroutine yields the goal until it is + * explicitly woken up via Worker::wakeUp. Wakeup can be queued from another + * thread via Worker::Waker. + */ + Co waitUntilWoken(); + Co waitForBuildSlot(); Co yield(); }; diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 7ce28d1deff4..ab09ef4a7e95 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -4,7 +4,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/store-api.hh" #include "nix/store/build/goal.hh" -#include "nix/util/muxable-pipe.hh" #include #include #include @@ -19,14 +18,14 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /** - * Whether to try to repair a valid path. + * Whether, if there are not substituters, to return ecNoSubstituters or ecFailed. */ - RepairFlag repair; + bool pathRequired; /** - * Pipe for the substituter's standard output. + * Whether to try to repair a valid path. */ - MuxablePipe outPipe; + RepairFlag repair; /** * The substituter thread. @@ -45,6 +44,7 @@ public: PathSubstitutionGoal( const StorePath & storePath, Worker & worker, + bool pathRequired, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); @@ -70,6 +70,8 @@ public: { return JobCategory::Substitution; }; + + Done doneFailure(ExitCode result, BuildResult::Failure failure); }; } // namespace nix diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index baf5c5a4d63e..8bb35221e72a 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -12,6 +12,7 @@ #include #include #include +#include namespace nix { @@ -91,6 +92,11 @@ private: */ WeakGoals wantingToBuild; + /** + * Goals waiting for a substitution slot. + */ + WeakGoals wantingToSubstitute; + /** * Child processes currently running. */ @@ -121,14 +127,14 @@ private: std::map> drvOutputSubstitutionGoals; /** - * Goals waiting for busy paths to be unlocked. + * Goals sleeping for a few seconds (polling a lock). */ - WeakGoals waitingForAnyGoal; + WeakGoals waitingForAWhile; /** - * Goals sleeping for a few seconds (polling a lock). + * Goals awaiting completion callbacks. */ - WeakGoals waitingForAWhile; + WeakGoals waitingForCompletion; /** * Last time the goals in `waitingForAWhile` were woken up. @@ -140,6 +146,42 @@ private: */ std::map pathContentsGoodCache; + class Waker + { +#ifndef _WIN32 + /** + * Wakeup pipe polled alongside all other goal FDs. Gets written to by + * enqueue(). Not needed on Windows. + */ + unix::SelfPipe wakeupPipe; +#else + Descriptor ioport; +#endif + /** + * Queue of goals that need to be woken up. + */ + Sync> wakeupQueue_; + + friend class Worker; + + void wakeAll(Worker & worker); + + Waker() + { +#ifndef _WIN32 + wakeupPipe.create(); +#endif + } + + public: + void enqueue(WeakGoalPtr goal); + }; + + /** + * This is behind a ref, so that other threads can take a weak_ptr to it. + */ + ref wakerState; + public: const Activity act; @@ -214,7 +256,7 @@ public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation); @@ -223,19 +265,22 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" */ std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); /** * @ref PathSubstitutionGoal "substitution goal" */ std::shared_ptr makePathSubstitutionGoal( - const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + const StorePath & storePath, + bool pathRequired = false, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id); /** @@ -256,6 +301,12 @@ public: */ void wakeUp(GoalPtr goal); + /** + * Get a weak reference to the goal waker. It can be used to safely enqueue Goals + * for wakeup from other threads. + */ + std::weak_ptr getCrossThreadWaker(); + /** * Return the number of local build processes currently running (but not * remote builds via the build hook). @@ -278,15 +329,12 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. `wakeSleepers` should be - * false if there is no sense in waking up goals that are sleeping - * because they can't run yet (e.g., there is no free build slot, - * or the hook would still say `postpone`). + * Unregisters a running child process. * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. */ - void childTerminated(Goal * goal, bool wakeSleepers = true); + void childTerminated(Goal * goal); /** * Unregisters a running child process, like the other overload. @@ -295,7 +343,7 @@ public: * weak goal references, so it is safe to call from destructors * where the goal object may be partially destroyed. */ - void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true); + void childTerminated(Goal * goal, JobCategory jobCategory); /** * Put `goal` to sleep until a build slot becomes available (which @@ -303,12 +351,6 @@ public: */ void waitForBuildSlot(GoalPtr goal); - /** - * Wait for any goal to finish. Pretty indiscriminate way to - * wait for some resource that some other goal is holding. - */ - void waitForAnyGoal(GoalPtr goal); - /** * Wait for a few seconds and then retry this goal. Used when * waiting for a lock held by another process. This kind of @@ -317,6 +359,11 @@ public: */ void waitForAWhile(GoalPtr goal); + /** + * Wait until explicitly resumed by Waker::enqueue. + */ + void waitForCompletion(GoalPtr goal); + /** * Loop until the specified top-level goals have finished. */ diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index e2caba3f1839..feeb6b49897c 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -8,8 +8,12 @@ # include "nix/store/aws-creds.hh" #endif +#include + namespace nix { +struct StructuredAttrs; + struct BuiltinBuilderContext { const BasicDerivation & drv; diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0f0b3f24b99..b528871b2801 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -25,6 +25,9 @@ struct Package class BuildEnvFileConflictError final : public CloneableError { +private: + void anchor() override; + public: const std::filesystem::path fileA; const std::filesystem::path fileB; diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c83b..d0eebe0bc23b 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 341d87b41667..375e1d0338d5 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -35,6 +35,7 @@ struct CommonProto struct ReadConn { Source & from; + bool shortStorePaths = false; }; /** @@ -44,6 +45,7 @@ struct CommonProto struct WriteConn { Sink & to; + bool shortStorePaths = false; }; template @@ -88,8 +90,9 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ /** * These use the empty string for the null case, relying on the fact diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 42b3415b777a..b622e82e26bf 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,12 +10,19 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; +private: + void anchor() override; + +public: + CommonSSHStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + { + } CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; + Setting> sshKey{ + this, std::nullopt, "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; Setting sshPublicHostKey{ this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 4e6ec22f9ccd..e29f660c4848 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include @@ -85,12 +84,12 @@ struct DerivationOptions * Either one set of checks for all outputs, or separate checks * per-output. */ - std::variant> outputChecks = OutputChecks{}; + std::variant>> outputChecks = OutputChecks{}; /** * Whether to avoid scanning for references for a given output. */ - std::map unsafeDiscardReferences; + std::map> unsafeDiscardReferences; /** * In non-structured mode, all bindings specified in the derivation @@ -123,7 +122,7 @@ struct DerivationOptions * attributes give to the builder. The set of paths in the original JSON * is replaced with a list of `PathInfo` in JSON format. */ - std::map> exportReferencesGraph; + std::map, std::less<>> exportReferencesGraph; /** * env: __sandboxProfile diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 0d137eaf36a2..56912981b4f7 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -17,6 +17,7 @@ namespace nix { struct StoreDirConfig; +struct Provenance; /* Abstract syntax of derivations. */ @@ -292,7 +293,7 @@ struct BasicDerivation BasicDerivation(const BasicDerivation &) = default; BasicDerivation & operator=(BasicDerivation &&) = default; BasicDerivation & operator=(const BasicDerivation &) = default; - virtual ~BasicDerivation() {}; + virtual ~BasicDerivation(); bool isBuiltin() const; @@ -427,6 +428,11 @@ struct Derivation : BasicDerivation void fillInOutputPaths(Store & store); Derivation() = default; + Derivation(Derivation &&) = default; + Derivation(const Derivation &) = default; + Derivation & operator=(Derivation &&) = default; + Derivation & operator=(const Derivation &) = default; + ~Derivation() override; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) @@ -507,34 +513,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, + using DrvHash = Hash; - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + /** + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. + */ + using CaOutputHashes = std::map; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. */ - Kind kind; -}; + using DeferredDrv = std::monostate; + + using Raw = std::variant; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -560,15 +571,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -583,7 +596,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; @@ -613,4 +626,5 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) +JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index c10af84ca26c..65be5df3c4fb 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -93,6 +93,19 @@ struct DerivedPathMap * `ChildNode::value`. */ ChildNode * findSlot(const SingleDerivedPath & k); + + /** + * Find the node for `k` and invoke @ref callback on it, pruning empty + * ancestors afterwards. + * + * @param callback Invoked on the found node. Should return true if + * the node's value is still non-empty (i.e. the node should be kept). + * If it returns false and the node has no children, the node is erased + * and empty ancestors are pruned recursively. + * + * No-op if the node does not exist. + */ + void removeSlot(const SingleDerivedPath & k, fun callback); }; template<> diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index ac7ab9c680fd..bec77a6bee68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; @@ -49,7 +53,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map> buildTrace; + boost::concurrent_flat_map> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index 59d1b1fddb22..c8a212c75603 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,8 +12,12 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Unix) { // Disable caching since this a temporary in-memory store. pathInfoCacheSize = 0; diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 15092202f1f6..4ea696f992f9 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -4,16 +4,11 @@ namespace nix { -/** - * Magic header of exportPath() output (obsolete). - */ -const uint32_t exportMagic = 0x4558494e; - /** * Export multiple paths in the format expected by `nix-store * --import`. The paths will be sorted topologically. */ -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink); +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version); /** * Import a sequence of NAR dumps created by `exportPaths()` into the diff --git a/src/libstore/include/nix/store/filetransfer-impl.hh b/src/libstore/include/nix/store/filetransfer-impl.hh new file mode 100644 index 000000000000..bac68baf55bd --- /dev/null +++ b/src/libstore/include/nix/store/filetransfer-impl.hh @@ -0,0 +1,73 @@ +#pragma once +/** + * @file + * + * Implementation details of filetransfer.cc exposed for unit testing. + * Not part of the public libstore API. + */ + +#include +#include +#include +#include +#include +#include + +namespace nix { + +/** + * Clamped exponential growth: base * 2^(attempt-1), capped at ceil. + * Shift is clamped at 31 and the intermediate is widened to uint64_t + * so the shift cannot overflow uint32_t. + */ +constexpr uint32_t clampedExponential(uint32_t base, uint32_t attempt, uint32_t ceil) +{ + auto shift = std::min(attempt == 0 ? 0u : attempt - 1, 31u); + uint64_t unclamped = static_cast(base) << shift; + return static_cast(std::min(unclamped, ceil)); +} + +/** + * Saturating conversion: chrono duration → uint32_t milliseconds. + * Negative inputs clamp to 0; values > UINT32_MAX clamp to UINT32_MAX. + */ +constexpr uint32_t saturateMs(std::chrono::milliseconds d) noexcept +{ + auto c = d.count(); + if (c <= 0) + return 0; + return static_cast(std::min(c, std::numeric_limits::max())); +} + +/** + * Parameters for computeRetryDelayMs. + */ +struct RetryDelayParams +{ + /** 1-based retry attempt number (1 = first retry). */ + uint32_t attempt; + /** Base delay in ms for this error class. */ + uint32_t baseMs; + /** Per-attempt delay ceiling (does not cap retryAfterMs). */ + uint32_t ceilMs; + /** Server-provided minimum delay (from Retry-After header). */ + std::optional retryAfterMs = {}; + /** Apply full jitter (false = deterministic). */ + bool jitter = true; +}; + +/** + * Compute the delay before the next retry attempt. + * + * Uses exponential backoff with optional full jitter. When a server-provided + * Retry-After is present, jitter spreads *above* it so that concurrent + * clients don't all retry at the same instant: + * sleep = random(floor, floor + backoff) + * where floor = retryAfter (or 0) and backoff = min(ceilMs, base * 2^(attempt-1)). + * ceilMs caps the backoff growth, not the server-provided floor. + * + * @param rng random number generator (unused if p.jitter is false) + */ +std::chrono::milliseconds computeRetryDelayMs(const RetryDelayParams & p, std::mt19937 & rng); + +} // namespace nix diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 688a6afcc6cc..606c075f3e0d 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include +#include #include #include @@ -21,16 +23,29 @@ namespace nix { const std::filesystem::path & nixConfDir(); -struct FileTransferSettings : Config +class FileTransferSettings : public Config { -private: - static std::filesystem::path getDefaultSSLCertFile(); + static std::optional getDefaultSSLCertFile(); + + void anchor() override; public: FileTransferSettings(); Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; + Setting enableHttp3{ + this, + false, + "http3", + R"( + Whether to try enabling HTTP/3 (QUIC). + When enabled, Nix requests HTTP/3 and transparently falls back + to HTTP/2 or HTTP/1.1 for servers that do not support it. + This option has no effect unless the `nix` binary is linked + against a libcurl built with HTTP/3 (QUIC) support. + )"}; + Setting userAgentSuffix{ this, "", "user-agent-suffix", "String appended to the user agent in HTTP requests."}; @@ -73,8 +88,63 @@ public: timeout's duration. )"}; - Setting tries{ - this, 5, "download-attempts", "The number of times Nix attempts to download a file before giving up."}; + Setting tries{ + this, + 5, + "filetransfer-retry-attempts", + R"( + The number of times Nix attempts a file transfer (download + or upload) before giving up. Retries apply to transient + failures: connection-level errors, HTTP 408, 429, and most + 5xx responses. Authentication failures (401/403/407), + 404/410, and other 4xx responses are not retried. + )", + {"download-attempts"}}; + + Setting retryDelayMs{ + this, + 100, + "filetransfer-retry-delay", + R"( + Initial delay in milliseconds before retrying a failed file transfer + (download or upload). The delay doubles with each subsequent attempt + (exponential backoff) and is subject to random jitter (see + `filetransfer-retry-jitter`). + )"}; + + Setting retryDelayRateLimitedMs{ + this, + 5000, + "filetransfer-retry-delay-rate-limited", + R"( + Initial delay in milliseconds before retrying a file transfer that + failed with a rate-limit response (HTTP 429 or 503). The delay doubles + with each subsequent attempt. + + Servers may send a `Retry-After` header specifying a longer delay; + when present, Nix respects the larger of the two values. + )"}; + + Setting retryMaxDelayMs{ + this, + 60000, + "filetransfer-retry-max-delay", + R"( + Ceiling on the exponential backoff delay in milliseconds. This does not + cap server-provided `Retry-After` values, which are honored as-is. + )"}; + + Setting retryJitter{ + this, + true, + "filetransfer-retry-jitter", + R"( + Whether to apply random jitter to retry delays. When enabled, each + retry waits for a random duration between 0 and the computed delay + ("full jitter"), which spreads out retry storms from many clients. + + Disable for deterministic retry timing (primarily useful for tests). + )"}; Setting downloadBufferSize{ this, @@ -147,8 +217,6 @@ public: extern FileTransferSettings fileTransferSettings; -extern const unsigned int RETRY_TIME_MS_DEFAULT; - /** * HTTP methods supported by FileTransfer. */ @@ -182,10 +250,19 @@ struct FileTransferRequest Headers headers; std::string expectedETag; HttpMethod method = HttpMethod::Get; - unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; + /** + * Per-request retry overrides. When set, these take precedence over the + * global `FileTransferSettings`. Typically populated from a store's URL + * parameters (e.g. `s3://bucket?retry-attempts=8`). + */ + std::optional retryDelayMs; + std::optional retryDelayRateLimitedMs; + std::optional retryMaxDelayMs; + std::optional retryAttempts; + /** * Optional path to the client certificate in "PEM" format. Only used for TLS-based protocols. */ @@ -243,6 +320,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ @@ -346,7 +431,7 @@ public: } }; - virtual ~FileTransfer() {} + virtual ~FileTransfer(); /** * Enqueue a data transfer request, returning a future to the result of @@ -354,14 +439,14 @@ public: * exception. */ virtual ItemHandle - enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; + enqueueFileTransfer(const FileTransferRequest & request, Callback callback) noexcept = 0; /** * Unpause a transfer that has been previously paused by a dataCallback. */ virtual void unpauseTransfer(ItemHandle handle) = 0; - std::future enqueueFileTransfer(const FileTransferRequest & request); + std::future enqueueFileTransfer(const FileTransferRequest & request) noexcept; /** * Synchronously download a file. @@ -385,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** @@ -403,8 +488,13 @@ ref getFileTransfer(); */ ref makeFileTransfer(const FileTransferSettings & settings = fileTransferSettings); +std::shared_ptr resetFileTransfer(); + class FileTransferError final : public CloneableError { +private: + void anchor() override; + public: FileTransfer::Error error; /// intentionally optional diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index de016e241de4..1e4e620eedca 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -7,9 +7,13 @@ namespace nix { +// FIXME: should turn this into an std::variant to represent the +// several root types. +using GcRootInfo = std::string; + typedef boost::unordered_flat_map< StorePath, - boost::unordered_flat_set>, + boost::unordered_flat_set>, std::hash> Roots; @@ -39,6 +43,19 @@ struct GCOptions using GCAction = nix::GCAction; using enum GCAction; + struct WholeStore + {}; + + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -50,14 +67,21 @@ struct GCOptions bool ignoreLiveness{false}; /** - * For `gcDeleteSpecific`, the paths to delete. + * The paths from which to delete. */ - StorePathSet pathsToDelete; + using GCPaths = std::variant; + GCPaths pathsToDelete; /** * Stop after at least `maxFreed` bytes have been freed. */ uint64_t maxFreed{std::numeric_limits::max()}; + + /** + * Whether to hide potentially sensitive information about GC + * roots (such as PIDs). + */ + bool censor = false; }; struct GCResults @@ -83,7 +107,7 @@ struct GCResults * * The notion of GC roots actually not part of this class. * - * - The base `Store` class has `Store::addTempRoot()` because for a store + * - The base `Store` class has `Store::addTempRoots()` because for a store * that doesn't support garbage collection at all, a temporary GC root is * safely implementable as no-op. * @@ -102,6 +126,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 3f2f122eb590..7841bc5949f9 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -15,17 +15,22 @@ namespace nix { +template<> +StoreReference BaseSetting::parse(const std::string & str) const; +template<> +std::string BaseSetting::to_string() const; + +template<> +std::set BaseSetting>::parse(const std::string & str) const; +template<> +std::string BaseSetting>::to_string() const; + struct ProfileDirsOptions; struct LogFileSettings : public virtual Config { - /** - * The directory where we log various operations. - */ - std::filesystem::path nixLogDir; - -protected: - LogFileSettings(); +private: + void anchor() override; public: Setting keepLog{ @@ -54,6 +59,10 @@ public: struct NarInfoDiskCacheSettings : public virtual Config { +private: + void anchor() override; + +public: Setting ttlNegative{ this, 3600, @@ -67,8 +76,8 @@ struct NarInfoDiskCacheSettings : public virtual Config To wipe the lookup cache completely: ```shell-session - $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* - # rm /root/.cache/nix/binary-cache-v*.sqlite* + $ rm $HOME/.cache/nix/binary-cache-*.sqlite* + # rm /root/.cache/nix/binary-cache-*.sqlite* ``` )"}; @@ -105,6 +114,9 @@ class Settings : public virtual Config, private WorkerSettings, private NarInfoDiskCacheSettings { +private: + void anchor() override; +public: StringSet getDefaultSystemFeatures(); StringSet getDefaultExtraPlatforms(); @@ -174,11 +186,6 @@ public: */ std::filesystem::path nixStateDir; - /** - * File name of the socket the daemon listens to. - */ - std::filesystem::path nixDaemonSocketFile; - Setting storeUri{ this, StoreReference::parse(getEnv("NIX_REMOTE").value_or("auto")), @@ -186,9 +193,14 @@ public: R"( The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. + See the [Store Types](@docroot@/store/types/index.md) section of the manual for supported store types and settings. + + Can be overridden by the [`NIX_REMOTE`](@docroot@/command-ref/env-common.md#env-NIX_REMOTE) environment variable. + + The default value is [`auto`](@docroot@/store/types/index.md#auto). )"}; Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; @@ -218,12 +230,8 @@ public: The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` - - `x86_64-darwin` - - `i686-linux` - `aarch64-linux` - `aarch64-darwin` - - `armv6l-linux` - - `armv7l-linux` In general, you do not have to modify this setting. While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. @@ -423,6 +431,8 @@ public: * Get the options needed for profile directory functions. */ ProfileDirsOptions getProfileDirsOptions() const; + + const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); }; // FIXME: don't use a global variable. @@ -446,6 +456,8 @@ void loadConfFile(AbstractConfig & config); */ extern std::string nixVersion; +extern const std::string determinateNixVersion; + /** * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. * @note When using libexpr, and/or libmain, This is not sufficient. See initNix(). diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 765eb6dd5135..12465261caef 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,7 +14,15 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this> tlsKey{ this, std::nullopt, "tls-private-key", "Path to an optional TLS client certificate private key in PEM format."}; + Setting retryDelayMs{ + this, + 0, + "retry-delay", + "Override [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay) for requests to this store (milliseconds)."}; + + Setting retryDelayRateLimitedMs{ + this, + 0, + "retry-delay-rate-limited", + "Override [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited) for requests to this store (milliseconds)."}; + + Setting retryMaxDelayMs{ + this, + 0, + "retry-max-delay", + "Override [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay) for requests to this store (milliseconds)."}; + + Setting retryAttempts{ + this, + 0, + "retry-attempts", + "Override [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) for requests to this store."}; + static const std::string name() { return "HTTP Binary Cache Store"; @@ -60,6 +92,8 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this getCompressionMethod(const std::string & path); diff --git a/src/libstore/include/nix/store/indirect-root-store.hh b/src/libstore/include/nix/store/indirect-root-store.hh index d477a320aa79..2f6a0fdb1ad2 100644 --- a/src/libstore/include/nix/store/indirect-root-store.hh +++ b/src/libstore/include/nix/store/indirect-root-store.hh @@ -38,6 +38,10 @@ namespace nix { */ struct IndirectRootStore : public virtual LocalFSStore { +private: + void anchor() override; + +public: inline static std::string operationName = "Indirect GC roots registration"; /** diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index acf755d7b9b0..40351014bf43 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -12,7 +12,15 @@ namespace nix { struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { - using CommonSSHStoreConfig::CommonSSHStoreConfig; +private: + void anchor() override; + +public: + LegacySSHStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Unix) + , CommonSSHStoreConfig(params) + { + } LegacySSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); @@ -59,6 +67,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; @@ -73,6 +85,11 @@ struct LegacySSHStore : public virtual Store ref openConnection(); + bool includeInProvenance() override + { + return true; + } + void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override; @@ -112,11 +129,12 @@ struct LegacySSHStore : public virtual Store StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, - ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, - HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair, + std::shared_ptr provenance) override { unsupported("addToStore"); } diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340f50..e1a80e8dc58e 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 8c4e68d26398..181b33e4bdf8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/store/binary-cache-store.hh" namespace nix { @@ -6,7 +9,15 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -22,7 +24,10 @@ private: } public: - using StoreConfig::StoreConfig; + LocalFSStoreConfig(const Params & params) + : StoreConfig(params, FilePathType::Native) + { + } /** * Used to override the `root` settings. Can't be done via modifying @@ -35,42 +40,48 @@ public: Setting> rootDir = makeRootDirSetting(*this, std::nullopt); -private: - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static std::filesystem::path getDefaultStateDir(); - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static std::filesystem::path getDefaultLogDir(); - -public: - Setting stateDir{ this, - rootDir.get() ? *rootDir.get() / "nix" / "var" / "nix" : getDefaultStateDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "nix" : StoreConfig::getStateDir(), "state", - "Directory where Nix stores state.", + R"( + Directory where Nix stores state. + + Defaults to [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) when [`root`](#@store-slug@-root) is not set. + )", }; Setting logDir{ this, - rootDir.get() ? *rootDir.get() / "nix" / "var" / "log" / "nix" : getDefaultLogDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "log" / "nix" : StoreConfig::getLogDir(), "log", - "directory where Nix stores log files.", + R"( + Directory where Nix stores log files. + + Defaults to [`NIX_LOG_DIR`](@docroot@/command-ref/env-common.md#env-NIX_LOG_DIR) when [`root`](#@store-slug@-root) is not set. + )", }; Setting realStoreDir{ this, rootDir.get() ? *rootDir.get() / "nix" / "store" : std::filesystem::path{storeDir}, "real", - "Physical path of the Nix store.", + R"( + Physical path of the Nix store. + + Defaults to [`store`](#@store-slug@-store) when [`root`](#@store-slug@-root) is not set. + )", }; + + const std::filesystem::path & getStateDir() const override + { + return stateDir.get(); + } + + const std::filesystem::path & getLogDir() const override + { + return logDir.get(); + } }; struct alignas(8) /* Work around ASAN failures on i686-linux. */ @@ -78,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 5d8ec1b4571a..10b04937c3aa 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,13 +7,17 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { } LocalOverlayStoreConfig(const std::filesystem::path & path, const Params & params) - : StoreConfig(params) + : StoreConfig(params, FilePathType::Native) , LocalFSStoreConfig(path, params) , LocalStoreConfig(path, params) { @@ -33,7 +37,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const Setting upperLayer{ (StoreConfig *) this, - "", + "/upper-layer-must-be-set", "upper-layer", R"( Directory containing the OverlayFS upper layer for this store's store dir. @@ -53,9 +57,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; - const Setting remountHook{ + const Setting> remountHook{ (StoreConfig *) this, - "", + std::nullopt, "remount-hook", R"( Script or other executable to run when overlay filesystem needs remounting. @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index af7eccdf44fd..88198802b66d 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -37,8 +37,20 @@ struct BaseSetting::trait template<> void BaseSetting::appendOrSet(PathsInChroot newValue, bool append); +template<> +std::map BaseSetting::toJSONObject() const; + +template<> +std::vector BaseSetting>::parse(const std::string & str) const; +template<> +std::string BaseSetting>::to_string() const; + struct GCSettings : public virtual Config { +private: + void anchor() override; + +public: Setting reservedSize{ this, 8 * 1024 * 1024, @@ -61,6 +73,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -80,6 +95,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; @@ -128,6 +146,10 @@ const uint32_t maxIdsPerBuild = struct AutoAllocateUidSettings : public virtual Config { +private: + void anchor() override; + +public: Setting startId{ this, #ifdef __linux__ @@ -163,6 +185,10 @@ struct AutoAllocateUidSettings : public virtual Config */ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllocateUidSettings { +private: + void anchor() override; + +public: /** * Get the GC settings. */ @@ -190,7 +216,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism.